--- - name: Ensure worker is not paused on deploy (remove .lock file) file: path: "{{ airflow_worker_dir }}/inputfiles/AIRFLOW.PREVENT_URL_PULL.lockfile" state: absent become: yes - name: Clean up old renamed lock files (older than 7 days) ansible.builtin.find: paths: "{{ airflow_worker_dir }}/inputfiles" patterns: "AIRFLOW.PREVENT_URL_PULL.lockfile.removed-*" age: "7d" use_regex: false register: old_lock_files become: yes - name: Remove found old lock files ansible.builtin.file: path: "{{ item.path }}" state: absent loop: "{{ old_lock_files.files }}" become: yes when: old_lock_files.files | length > 0 - name: Ensure YT-DLP worker inputfiles directory exists file: path: "{{ airflow_worker_dir }}/inputfiles" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0755' become: yes - name: Ensure YT-DLP worker logs directory exists file: path: "{{ airflow_worker_dir }}/logs" state: directory owner: "{{ airflow_uid }}" group: "{{ deploy_group }}" mode: '0775' become: yes - name: Check if YT-DLP worker deployment directory exists stat: path: "{{ airflow_worker_dir }}" register: worker_dir_stat - name: Ensure YT-DLP worker deployment directory exists file: path: "{{ airflow_worker_dir }}" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0755' become: yes when: not worker_dir_stat.stat.exists - name: Ensure YT-DLP worker configs directory exists file: path: "{{ airflow_worker_dir }}/configs" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0755' become: yes - name: "Log: Syncing YT-DLP service files" debug: msg: "Syncing YT-DLP service components (config generator, envoy/camoufox templates) to the worker node." - name: Sync YT-DLP service files to worker synchronize: src: "../{{ item }}" dest: "{{ airflow_worker_dir }}/" archive: yes recursive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: - "airflow/camoufox" - name: Sync YT-DLP config generator to worker synchronize: src: "../airflow/generate_envoy_config.py" dest: "{{ airflow_worker_dir }}/" archive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" - name: Sync YT-DLP config files to worker synchronize: src: "../airflow/configs/{{ item }}" dest: "{{ airflow_worker_dir }}/configs/" archive: yes recursive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: - "docker-compose-ytdlp-ops.yaml.j2" - "docker-compose.config-generate.yaml" - "envoy.yaml.j2" - "docker-compose.camoufox.yaml.j2" - name: Sync Airflow build context to worker synchronize: src: "../{{ item }}" dest: "{{ airflow_worker_dir }}/" archive: yes recursive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: - "airflow/Dockerfile" - "setup.py" - "VERSION" - "yt_ops_services" - "thrift_model" - "pangramia" - name: Create .env file for YT-DLP worker service template: src: "../../templates/.env.j2" dest: "{{ airflow_worker_dir }}/.env" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: "{{ deploy_group }}" become: yes vars: service_role: "worker" server_identity: "ytdlp-ops-service-worker-{{ inventory_hostname }}" - name: Create symlink for .env in configs directory for manual docker-compose commands file: src: "../.env" dest: "{{ airflow_worker_dir }}/configs/.env" state: link force: yes owner: "{{ ssh_user }}" group: "{{ deploy_group }}" become: yes - name: Log in to Docker Hub to pull private images community.docker.docker_login: username: "{{ dockerhub_user }}" password: "{{ vault_dockerhub_password }}" when: vault_dockerhub_password is defined and vault_dockerhub_password | length > 0 - name: "Log: Generating YT-DLP service configurations" debug: msg: "Running the configuration generator script inside a temporary Docker container. This creates docker-compose, envoy, and camoufox files based on .env variables." - name: Ensure previously generated config files are removed before generation file: path: "{{ item }}" state: absent loop: - "{{ airflow_worker_dir }}/envoy.yaml" - "{{ airflow_worker_dir }}/configs/docker-compose.camoufox.yaml" - "{{ airflow_worker_dir }}/configs/camoufox_endpoints.json" become: yes - name: Create placeholder envoy.yaml to prevent Docker from creating a directory file: path: "{{ airflow_worker_dir }}/envoy.yaml" state: touch owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0664' become: yes - name: Generate YT-DLP service configurations shell: cmd: "docker compose --project-directory {{ airflow_worker_dir }} -f configs/docker-compose.config-generate.yaml run --rm config-generator" chdir: "{{ airflow_worker_dir }}" become: yes become_user: "{{ ssh_user }}" - name: Clean up old root docker-compose files to prevent conflicts ansible.builtin.file: path: "{{ airflow_worker_dir }}/{{ item }}" state: absent loop: - "docker-compose.yml" - "docker-compose.yaml" - "docker-compose.override.yml" - "docker-compose.airflow.yml" become: yes - name: Template docker-compose file for Airflow worker template: src: "{{ playbook_dir }}/../airflow/configs/docker-compose-dl.yaml.j2" dest: "{{ airflow_worker_dir }}/configs/docker-compose.airflow.yml" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: "{{ deploy_group }}" become: yes - name: "Log: Building Airflow image" debug: msg: "Building the Airflow image locally. This image contains all dependencies for running DAGs." - name: Build Airflow image from local Dockerfile community.docker.docker_image: name: "pangramia/ytdlp-ops-airflow:latest" build: path: "{{ airflow_worker_dir }}" dockerfile: "Dockerfile" source: build force_source: true when: not fast_deploy | default(false) - name: "Log: Building Camoufox (remote browser) image" debug: msg: "Building the Camoufox image locally. This image provides remote-controlled Firefox browsers for token generation." - name: Build Camoufox image from local Dockerfile community.docker.docker_image: name: "camoufox:latest" build: path: "{{ airflow_worker_dir }}/camoufox" source: build force_source: true when: not fast_deploy | default(false) - name: Ensure correct permissions for build context after generation file: path: "{{ airflow_worker_dir }}" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" recurse: yes become: yes - name: Check for shadowsocks-rust proxy compose file stat: path: "/srv/shadowsocks-rust/docker-compose.proxies.yaml" register: proxy_compose_file - name: "Log: Stopping worker services before start" debug: msg: "Stopping all worker services to ensure a clean start." - name: Stop all worker services community.docker.docker_compose_v2: project_src: "{{ airflow_worker_dir }}" files: - "configs/docker-compose-ytdlp-ops.yaml" - "configs/docker-compose.camoufox.yaml" - "configs/docker-compose.airflow.yml" state: absent remove_volumes: true # Corresponds to docker compose down -v - name: Forcefully remove project-specific Docker volumes to fix corruption issues ansible.builtin.shell: "docker volume ls -q --filter 'label=com.docker.compose.project=ytdlp-ops-worker' | xargs -r docker volume rm --force" become: yes register: removed_volumes changed_when: removed_volumes.stdout | length > 0 failed_when: false - name: "Log: Starting all worker services" debug: msg: "Starting all worker services: ytdlp-ops, camoufox, and airflow-worker." - name: Start all worker services community.docker.docker_compose_v2: project_src: "{{ airflow_worker_dir }}" files: - "configs/docker-compose-ytdlp-ops.yaml" - "configs/docker-compose.camoufox.yaml" - "configs/docker-compose.airflow.yml" state: present remove_orphans: true pull: "{{ 'never' if fast_deploy | default(false) else 'missing' }}" recreate: always # Corresponds to --force-recreate - name: Include camoufox verification tasks include_tasks: ../../../tasks/verify_camoufox.yml when: not fast_deploy | default(false)