#!/bin/bash # # Syncs the project directory to a remote "jump" host for deployment orchestration. # # This script is designed to be run from the root of the project directory. # It syncs essential project files like source code, DAGs, and Ansible playbooks, # while excluding generated files, local data, logs, and other non-essential files # to ensure a clean copy is deployed. set -e # Exit immediately if a command exits with a non-zero status. set -u # Treat unset variables as an error. # --- Configuration --- # IMPORTANT: Update these variables to match your environment. # # The remote host to sync to (e.g., user@hostname) REMOTE_HOST="alex_p@af-jump" # The destination path on the remote host REMOTE_PATH="/home/alex_p/yt-ops-services" # The root directory of the project on the local machine. SOURCE_DIR="." # --- rsync command --- echo ">>> Syncing project from '$SOURCE_DIR' to '$REMOTE_HOST:$REMOTE_PATH'..." # Use an array for exclude options for clarity and to handle spaces correctly. # This list is based on an analysis of the project structure and generated artifacts. EXCLUDE_OPTS=( "--exclude=.git" "--exclude=__pycache__" "--exclude='*.pyc'" "--exclude='*.log'" "--exclude=.DS_Store" "--exclude=.vault_pass" "--exclude=.env" "--exclude=ansible/inventory.ini" "--exclude=ansible/host_vars/" "--exclude=ansible/group_vars/all/generated_vars.yml" "--exclude=postgres-data/" "--exclude=redis-data/" "--exclude=minio-data/" "--exclude=logs/" "--exclude=downloadfiles/" "--exclude=addfiles/" "--exclude=token_generator/node_modules/" # Exclude files generated on remote hosts by Ansible/config-generator "--exclude=airflow/configs/envoy.yaml" "--exclude=airflow/configs/docker-compose.camoufox.yaml" "--exclude=airflow/configs/camoufox_endpoints.json" "--exclude=cluster*.yml" # Exclude local development notes "--exclude=TODO-*.md" # Exclude user-specific tools "--exclude=*aider*" ) # The rsync command: # -a: archive mode (recursive, preserves permissions, etc.) # -v: verbose # -z: compress file data during the transfer # --delete: delete extraneous files from the destination directory # --partial: keep partially transferred files # --progress: show progress during transfer rsync -avz --delete --partial --progress \ "${EXCLUDE_OPTS[@]}" \ "$SOURCE_DIR/" \ "$REMOTE_HOST:$REMOTE_PATH/" echo ">>> Sync complete."