--- - name: Ensure worker is not paused on deploy (remove .lock file) file: path: "{{ airflow_worker_dir }}/inputfiles/AIRFLOW.PREVENT_URL_PULL.lockfile" state: absent become: yes - name: Clean up old renamed lock files (older than 7 days) ansible.builtin.find: paths: "{{ airflow_worker_dir }}/inputfiles" patterns: "AIRFLOW.PREVENT_URL_PULL.lockfile.removed-*" age: "7d" use_regex: false register: old_lock_files become: yes - name: Remove found old lock files ansible.builtin.file: path: "{{ item.path }}" state: absent loop: "{{ old_lock_files.files }}" become: yes when: old_lock_files.files | length > 0 - name: Ensure YT-DLP worker inputfiles directory exists file: path: "{{ airflow_worker_dir }}/inputfiles" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0755' become: yes - name: Ensure YT-DLP worker logs directory exists file: path: "{{ airflow_worker_dir }}/logs" state: directory owner: "{{ airflow_uid }}" group: "{{ deploy_group }}" mode: '0775' become: yes - name: Check if YT-DLP worker deployment directory exists stat: path: "{{ airflow_worker_dir }}" register: worker_dir_stat - name: Ensure YT-DLP worker deployment directory exists file: path: "{{ airflow_worker_dir }}" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0755' become: yes when: not worker_dir_stat.stat.exists - name: Ensure YT-DLP worker configs directory exists file: path: "{{ airflow_worker_dir }}/configs" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0755' become: yes - name: "Log: Syncing YT-DLP service files" debug: msg: "Syncing YT-DLP service components (config generator, envoy/camoufox templates) to the worker node." - name: Sync YT-DLP service files to worker synchronize: src: "../{{ item }}" dest: "{{ airflow_worker_dir }}/" archive: yes recursive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: - "airflow/camoufox" - name: Sync YT-DLP config generator to worker synchronize: src: "../airflow/generate_envoy_config.py" dest: "{{ airflow_worker_dir }}/" archive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" - name: Sync YT-DLP config files to worker synchronize: src: "../airflow/configs/{{ item }}" dest: "{{ airflow_worker_dir }}/configs/" archive: yes recursive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: - "docker-compose-ytdlp-ops.yaml.j2" - "docker-compose.config-generate.yaml" - "envoy.yaml.j2" - "docker-compose.camoufox.yaml.j2" - name: Create .env file for YT-DLP worker service template: src: "../../templates/.env.j2" dest: "{{ airflow_worker_dir }}/.env" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: "{{ deploy_group }}" become: yes vars: service_role: "worker" server_identity: "ytdlp-ops-service-worker-{{ inventory_hostname }}" - name: Create symlink for .env in configs directory for manual docker-compose commands file: src: "../.env" dest: "{{ airflow_worker_dir }}/configs/.env" state: link force: yes owner: "{{ ssh_user }}" group: "{{ deploy_group }}" become: yes - name: Log in to Docker Hub to pull private images community.docker.docker_login: username: "{{ dockerhub_user }}" password: "{{ vault_dockerhub_password }}" when: vault_dockerhub_password is defined and vault_dockerhub_password | length > 0 - name: "Log: Generating YT-DLP service configurations" debug: msg: "Running the configuration generator script inside a temporary Docker container. This creates docker-compose, envoy, and camoufox files based on .env variables." - name: Ensure previously generated config files are removed before generation file: path: "{{ item }}" state: absent loop: - "{{ airflow_worker_dir }}/envoy.yaml" - "{{ airflow_worker_dir }}/configs/docker-compose.camoufox.yaml" - "{{ airflow_worker_dir }}/configs/camoufox_endpoints.json" become: yes - name: Create placeholder envoy.yaml to prevent Docker from creating a directory file: path: "{{ airflow_worker_dir }}/envoy.yaml" state: touch owner: "{{ ssh_user }}" group: "{{ deploy_group }}" mode: '0664' become: yes - name: Generate YT-DLP service configurations shell: cmd: "docker compose --project-directory {{ airflow_worker_dir }} -f configs/docker-compose.config-generate.yaml run --rm config-generator" chdir: "{{ airflow_worker_dir }}" become: yes become_user: "{{ ssh_user }}" - name: Clean up old root docker-compose files to prevent conflicts ansible.builtin.file: path: "{{ airflow_worker_dir }}/{{ item }}" state: absent loop: - "docker-compose.yml" - "docker-compose.yaml" - "docker-compose.override.yml" - "docker-compose.airflow.yml" become: yes - name: Template docker-compose file for Airflow worker template: src: "{{ playbook_dir }}/../airflow/configs/docker-compose-dl.yaml.j2" dest: "{{ airflow_worker_dir }}/configs/docker-compose.airflow.yml" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: "{{ deploy_group }}" become: yes - name: "Log: Building Camoufox (remote browser) image" debug: msg: "Building the Camoufox image locally. This image provides remote-controlled Firefox browsers for token generation." - name: Build Camoufox image from local Dockerfile community.docker.docker_image: name: "camoufox:latest" build: path: "{{ airflow_worker_dir }}/camoufox" source: build force_source: true when: not fast_deploy | default(false) - name: Ensure correct permissions for build context after generation file: path: "{{ airflow_worker_dir }}" state: directory owner: "{{ ssh_user }}" group: "{{ deploy_group }}" recurse: yes become: yes - name: Check for shadowsocks-rust proxy compose file stat: path: "/srv/shadowsocks-rust/docker-compose.proxies.yaml" register: proxy_compose_file - name: "Log: Starting all worker services" debug: msg: "Starting all worker services: ytdlp-ops, camoufox, and airflow-worker." - name: Start all worker services community.docker.docker_compose_v2: project_src: "{{ airflow_worker_dir }}" files: - "configs/docker-compose-ytdlp-ops.yaml" - "configs/docker-compose.camoufox.yaml" - "configs/docker-compose.airflow.yml" state: present remove_orphans: true pull: "{{ 'never' if fast_deploy | default(false) else 'missing' }}" - name: Include camoufox verification tasks include_tasks: ../../../tasks/verify_camoufox.yml when: not fast_deploy | default(false)