--- - name: Check if Airflow worker deployment directory exists stat: path: "{{ airflow_worker_dir }}" register: worker_dir_stat - name: Ensure Airflow worker deployment directory exists file: path: "{{ airflow_worker_dir }}" state: directory owner: "{{ ssh_user }}" group: ytdl mode: '0755' become: yes when: not worker_dir_stat.stat.exists - name: "Log: Syncing Airflow core files" debug: msg: "Syncing DAGs, configs, and Python source code to the worker node." - name: Sync Airflow worker files synchronize: src: "../{{ item }}" dest: "{{ airflow_worker_dir }}/" archive: yes recursive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: - "airflow/Dockerfile" - "airflow/.dockerignore" - "airflow/dags" - "airflow/config" - "yt_ops_package/setup.py" - "yt_ops_package/yt_ops_services" - "yt_ops_package/thrift_model" - "yt_ops_package/VERSION" - "yt_ops_package/pangramia" - "airflow/init-airflow.sh" - "airflow/update-yt-dlp.sh" - "yt_ops_package/get_info_json_client.py" - "yt_ops_package/proxy_manager_client.py" - "token_generator" - "utils" - name: Check if source directories exist stat: path: "../{{ item }}" register: source_dirs loop: - "airflow/inputfiles" - "airflow/plugins" - "airflow/addfiles" - "airflow/bgutil-ytdlp-pot-provider" - name: Sync optional directories if they exist synchronize: src: "../{{ item.item }}/" dest: "{{ airflow_worker_dir }}/{{ item.item | basename }}/" archive: yes recursive: yes delete: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: "{{ source_dirs.results }}" when: item.stat.exists - name: Sync pangramia thrift files synchronize: src: "../yt_ops_package/thrift_model/gen_py/pangramia/" dest: "{{ airflow_worker_dir }}/pangramia/" archive: yes recursive: yes delete: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" - name: Template docker-compose file for worker template: src: "{{ playbook_dir }}/../airflow/docker-compose-dl.yaml.j2" dest: "{{ airflow_worker_dir }}/docker-compose-dl.yaml" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: ytdl become: yes - name: Create .env file for Airflow worker service template: src: "../../templates/.env.worker.j2" dest: "{{ airflow_worker_dir }}/.env" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: ytdl become: yes vars: service_role: "worker" - name: Create symlink for docker-compose.yaml file: src: "{{ airflow_worker_dir }}/docker-compose-dl.yaml" dest: "{{ airflow_worker_dir }}/docker-compose.yaml" state: link owner: "{{ ssh_user }}" group: ytdl follow: no - name: Ensure correct permissions for build context file: path: "{{ airflow_worker_dir }}" state: directory owner: "{{ ssh_user }}" group: ytdl recurse: yes become: yes - name: Verify Dockerfile exists in build directory stat: path: "{{ airflow_worker_dir }}/Dockerfile" register: dockerfile_stat - name: Fail if Dockerfile is missing fail: msg: "Dockerfile not found in {{ airflow_worker_dir }}. Cannot build image." when: not dockerfile_stat.stat.exists - name: "Log: Building Airflow Docker image" debug: msg: "Building the main Airflow Docker image ({{ airflow_image_name }}) locally on the worker node. This may take a few minutes." - name: Build Airflow worker image community.docker.docker_image: name: "{{ airflow_image_name }}" build: path: "{{ airflow_worker_dir }}" dockerfile: "Dockerfile" source: build force_source: true - name: Make Airflow init script executable file: path: "{{ airflow_worker_dir }}/init-airflow.sh" mode: "0755" become: yes - name: Run Airflow init script shell: cmd: "./init-airflow.sh" chdir: "{{ airflow_worker_dir }}" become: yes become_user: "{{ ssh_user }}" - name: "Log: Starting Airflow services" debug: msg: "Starting Airflow worker services (celery worker) on the node using docker-compose." - name: Start Airflow worker service community.docker.docker_compose_v2: project_src: "{{ airflow_worker_dir }}" files: - "docker-compose-dl.yaml" state: present remove_orphans: true