#!/bin/bash set -euo pipefail # --- Environment Setup --- ENV="" # Parse command-line arguments if [[ "$#" -gt 0 && "$1" == "--env" ]]; then if [[ -n "$2" && ("$2" == "prod" || "$2" == "test") ]]; then ENV="$2" else echo "Error: Invalid environment specified for deploy-dl.sh. Use 'prod' or 'test'." >&2 exit 1 fi else echo "Usage: $0 --env [prod|test]" >&2 exit 1 fi # --- Configuration --- SSH_USER="alex_p" if [[ "$ENV" == "prod" ]]; then WORKER_SERVERS=("dl003") elif [[ "$ENV" == "test" ]]; then WORKER_SERVERS=("dl001") fi REMOTE_DEST_PATH="/srv/airflow_dl_worker/" # List of files and directories to sync from the project root. # This script assumes it is run from the project root via deploy_all.sh ROOT_FILES_TO_SYNC=( "Dockerfile" "get_info_json_client.py" "proxy_manager_client.py" "setup.py" "VERSION" "generate_tokens_direct.mjs" ) AIRFLOW_FILES_TO_SYNC=( "docker-compose-ytdlp-ops.yaml" "init-airflow.sh" ) DIRS_TO_SYNC=( "airflow/camoufox/" "airflow/inputfiles/" "server_fix/" "token_generator/" "utils/" "yt_ops_services/" ) RSYNC_OPTS="-avz --progress --delete --exclude='__pycache__/' --exclude='*.pyc' --exclude='*.pyo' --exclude='node_modules/'" echo ">>> Deploying to DL WORKER(S) for environment: $ENV" # --- Deployment --- for worker in "${WORKER_SERVERS[@]}"; do WORKER_HOST="${SSH_USER}@${worker}" echo "--------------------------------------------------" echo ">>> Deploying to WORKER: $WORKER_HOST" echo "--------------------------------------------------" echo ">>> Creating remote directory on WORKER: $WORKER_HOST" ssh "$WORKER_HOST" "mkdir -p $REMOTE_DEST_PATH" echo ">>> Syncing individual files to WORKER..." for f in "${ROOT_FILES_TO_SYNC[@]}"; do echo " - Syncing $f" rsync $RSYNC_OPTS "$f" "$WORKER_HOST:$REMOTE_DEST_PATH" done for f in "${AIRFLOW_FILES_TO_SYNC[@]}"; do echo " - Syncing airflow/$f" rsync $RSYNC_OPTS "airflow/$f" "$WORKER_HOST:$REMOTE_DEST_PATH" done echo ">>> Syncing directories to WORKER..." for d in "${DIRS_TO_SYNC[@]}"; do echo " - Syncing $d" rsync $RSYNC_OPTS "$d" "$WORKER_HOST:$REMOTE_DEST_PATH" done echo ">>> Renaming worker compose file on remote..." ssh "$WORK_HOST" "cd $REMOTE_DEST_PATH && ln -sf docker-compose-ytdlp-ops.yaml docker-compose.yaml" done echo ">>> DL WORKER(S) deployment sync complete." exit 0