--- - name: Check if Airflow master deployment directory exists stat: path: "{{ airflow_master_dir }}" register: master_dir_stat - name: Ensure Airflow master deployment directory exists file: path: "{{ airflow_master_dir }}" state: directory owner: "{{ ssh_user }}" group: ytdl mode: '0755' become: yes when: not master_dir_stat.stat.exists - name: Check if source directories exist stat: path: "../{{ item }}" register: source_dirs loop: - "airflow/inputfiles" - "airflow/plugins" - "airflow/addfiles" - "airflow/bgutil-ytdlp-pot-provider" - name: "Log: Syncing Airflow core files" debug: msg: "Syncing DAGs, configs, and Python source code to the master node." - name: Sync Airflow master files synchronize: src: "../{{ item }}" dest: "{{ airflow_master_dir }}/" archive: yes recursive: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: - "airflow/Dockerfile" - "airflow/.dockerignore" - "airflow/docker-compose-master.yaml" - "airflow/dags" - "airflow/config" - "setup.py" - "yt_ops_services" - "thrift_model" - "VERSION" - "airflow/init-airflow.sh" - "airflow/update-yt-dlp.sh" - "airflow/nginx.conf" - "get_info_json_client.py" - "proxy_manager_client.py" - "token_generator" - "utils" - name: Sync optional directories if they exist synchronize: src: "../{{ item.item }}/" dest: "{{ airflow_master_dir }}/{{ item.item | basename }}/" archive: yes recursive: yes delete: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" loop: "{{ source_dirs.results }}" when: item.stat.exists - name: Sync pangramia thrift files synchronize: src: "../thrift_model/gen_py/pangramia/" dest: "{{ airflow_master_dir }}/pangramia/" archive: yes recursive: yes delete: yes rsync_path: "sudo rsync" rsync_opts: "{{ rsync_default_opts }}" - name: Create .env file for Airflow master service template: src: "../../templates/.env.master.j2" dest: "{{ airflow_master_dir }}/.env" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: ytdl become: yes vars: service_role: "master" - name: Template Minio connection file template: src: "../airflow/config/minio_default_conn.json.j2" dest: "{{ airflow_master_dir }}/config/minio_default_conn.json" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: ytdl become: yes - name: Template YT-DLP Redis connection file template: src: "../airflow/config/ytdlp_redis_conn.json.j2" dest: "{{ airflow_master_dir }}/config/ytdlp_redis_conn.json" mode: "{{ file_permissions }}" owner: "{{ ssh_user }}" group: ytdl become: yes - name: Create symlink for docker-compose.yaml file: src: "{{ airflow_master_dir }}/docker-compose-master.yaml" dest: "{{ airflow_master_dir }}/docker-compose.yaml" state: link owner: "{{ ssh_user }}" group: ytdl force: yes follow: no - name: Ensure correct permissions for build context file: path: "{{ airflow_master_dir }}" state: directory owner: "{{ ssh_user }}" group: ytdl recurse: yes become: yes - name: Verify Dockerfile exists in build directory stat: path: "{{ airflow_master_dir }}/Dockerfile" register: dockerfile_stat - name: Fail if Dockerfile is missing fail: msg: "Dockerfile not found in {{ airflow_master_dir }}. Cannot build image." when: not dockerfile_stat.stat.exists - name: "Log: Building Airflow Docker image" debug: msg: "Building the main Airflow Docker image ({{ airflow_image_name }}) locally on the master node. This may take a few minutes." - name: Build Airflow master image community.docker.docker_image: name: "{{ airflow_image_name }}" build: path: "{{ airflow_master_dir }}" dockerfile: "Dockerfile" # Explicitly specify the Dockerfile name source: build force_source: true - name: Make Airflow init script executable file: path: "{{ airflow_master_dir }}/init-airflow.sh" mode: "0755" become: yes - name: Run Airflow init script shell: cmd: "./init-airflow.sh" chdir: "{{ airflow_master_dir }}" become: yes become_user: "{{ ssh_user }}" - name: "Log: Starting Airflow services" debug: msg: "Starting Airflow core services (webserver, scheduler, etc.) on the master node using docker-compose." - name: Start Airflow master service community.docker.docker_compose_v2: project_src: "{{ airflow_master_dir }}" files: - "docker-compose-master.yaml" state: present remove_orphans: true