yt-dlp-dags/ansible/roles/airflow-master/tasks/main.yml

---
- name: Check if Airflow master deployment directory exists
  stat:
    path: "{{ airflow_master_dir }}"
  register: master_dir_stat

- name: Ensure Airflow master deployment directory exists
  file:
    path: "{{ airflow_master_dir }}"
    state: directory
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    mode: '0755'
  become: yes
  when: not master_dir_stat.stat.exists

- name: Ensure Airflow master configs directory exists
  file:
    path: "{{ airflow_master_dir }}/configs"
    state: directory
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    mode: '0755'
  become: yes

- name: Ensure Airflow master config directory exists
  file:
    path: "{{ airflow_master_dir }}/config"
    state: directory
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    mode: '0755'
  become: yes

- name: Ensure Airflow operational directories exist with correct permissions
  file:
    path: "{{ airflow_master_dir }}/{{ item }}"
    state: directory
    owner: "{{ airflow_uid }}"
    group: "{{ deploy_group }}"
    mode: '0775'
  become: yes
  loop:
    - "dags"
    - "logs"
    - "plugins"
    - "downloadfiles"
    - "addfiles"
    - "inputfiles"

- name: Check if source directories exist
  stat:
    path: "../{{ item }}"
  register: source_dirs
  loop:
    - "airflow/plugins"
    - "airflow/addfiles"
    - "airflow/bgutil-ytdlp-pot-provider"

- name: "Log: Syncing Airflow core files"
  debug:
    msg: "Syncing DAGs, configs, and Python source code to the master node."

- name: Sync Airflow master files
  synchronize:
    src: "../{{ item }}"
    dest: "{{ airflow_master_dir }}/"
    archive: yes
    recursive: yes
    delete: yes
    rsync_path: "sudo rsync"
    rsync_opts: "{{ rsync_default_opts }}"
  loop:
    - "airflow/Dockerfile"
    - "airflow/Dockerfile.caddy"
    - "airflow/.dockerignore"
    - "airflow/dags"
    - "airflow/inputfiles"
    - "setup.py"
    - "yt_ops_services"
    - "thrift_model"
    - "VERSION"
    - "airflow/update-yt-dlp.sh"
    - "proxy_manager_client.py"
    - "utils"

- name: Copy custom Python config files to master
  copy:
    src: "../airflow/config/{{ item }}"
    dest: "{{ airflow_master_dir }}/config/{{ item }}"
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    mode: '0644'
  become: yes
  loop:
    - "custom_task_hooks.py"
    - "airflow_local_settings.py"

- name: Ensure any existing airflow.cfg directory is removed
  file:
    path: "{{ airflow_master_dir }}/config/airflow.cfg"
    state: absent
  become: yes
  ignore_errors: yes

- name: Copy airflow.cfg to master
  copy:
    src: "../airflow/airflow.cfg"
    dest: "{{ airflow_master_dir }}/config/airflow.cfg"
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    mode: '0644'
  become: yes

- name: Sync Airflow master config files
  synchronize:
    src: "../airflow/configs/{{ item }}"
    dest: "{{ airflow_master_dir }}/configs/"
    archive: yes
    recursive: yes
    rsync_path: "sudo rsync"
    rsync_opts: "{{ rsync_default_opts }}"
  loop:
    - "nginx.conf"
    - "Caddyfile"

- name: Sync optional directories if they exist
  synchronize:
    src: "../{{ item.item }}/"
    dest: "{{ airflow_master_dir }}/{{ item.item | basename }}/"
    archive: yes
    recursive: yes
    delete: yes
    rsync_path: "sudo rsync"
    rsync_opts: "{{ rsync_default_opts }}"
  loop: "{{ source_dirs.results }}"
  when: item.stat.exists

- name: Sync pangramia thrift files
  synchronize:
    src: "../thrift_model/gen_py/pangramia/"
    dest: "{{ airflow_master_dir }}/pangramia/"
    archive: yes
    recursive: yes
    delete: yes
    rsync_path: "sudo rsync"
    rsync_opts: "{{ rsync_default_opts }}"

- name: Template docker-compose file for master
  template:
    src: "{{ playbook_dir }}/../airflow/configs/docker-compose-master.yaml.j2"
    dest: "{{ airflow_master_dir }}/configs/docker-compose-master.yaml"
    mode: "{{ file_permissions }}"
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
  become: yes

- name: Template Redis connection file
  template:
    src: "../airflow/config/redis_default_conn.json.j2"
    dest: "{{ airflow_master_dir }}/config/redis_default_conn.json"
    mode: "{{ file_permissions }}"
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
  become: yes

- name: Template Minio connection file for master
  template:
    src: "../airflow/config/minio_default_conn.json.j2"
    dest: "{{ airflow_master_dir }}/config/minio_default_conn.json"
    mode: "{{ file_permissions }}"
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
  become: yes

- name: Ensure config directory is group-writable for Airflow initialization
  file:
    path: "{{ airflow_master_dir }}/config"
    state: directory
    mode: '0775'
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
  become: yes

- name: Ensure airflow.cfg is group-writable for Airflow initialization
  file:
    path: "{{ airflow_master_dir }}/config/airflow.cfg"
    state: file
    mode: '0664'
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
  become: yes

- name: Create symlink for docker-compose.yaml
  file:
    src: "{{ airflow_master_dir }}/configs/docker-compose-master.yaml"
    dest: "{{ airflow_master_dir }}/docker-compose.yaml"
    state: link
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    force: yes
    follow: no

- name: Ensure correct permissions for build context
  file:
    path: "{{ airflow_master_dir }}"
    state: directory
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    recurse: yes
  become: yes

- name: Ensure postgres-data directory exists on master and has correct permissions
  file:
    path: "{{ airflow_master_dir }}/postgres-data"
    state: directory
    owner: "999"  # UID for the 'postgres' user in the official postgres image
    group: "999"  # GID for the 'postgres' group in the official postgres image
    mode: '0700'
    recurse: yes
  become: yes

- name: Set proper ownership and permissions on master logs directory contents
  shell: |
    chown -R {{ airflow_uid }}:{{ deploy_group }} {{ airflow_master_dir }}/logs
    find {{ airflow_master_dir }}/logs -type d -exec chmod g+rws {} +
    find {{ airflow_master_dir }}/logs -type f -exec chmod g+rw {} +
  become: yes

- name: Verify Dockerfile exists in build directory
  stat:
    path: "{{ airflow_master_dir }}/Dockerfile"
  register: dockerfile_stat

- name: Fail if Dockerfile is missing
  fail:
    msg: "Dockerfile not found in {{ airflow_master_dir }}. Cannot build image."
  when: not dockerfile_stat.stat.exists

- name: "Log: Building Airflow Docker image"
  debug:
    msg: "Building the main Airflow Docker image ({{ airflow_image_name }}) locally on the master node. This may take a few minutes."

- name: Build Airflow master image
  community.docker.docker_image:
    name: "{{ airflow_image_name }}"
    build:
      path: "{{ airflow_master_dir }}"
      dockerfile: "Dockerfile"  # Explicitly specify the Dockerfile name
    source: build
    force_source: true
  when: not fast_deploy | default(false)

- name: "Log: Preparing assets for Caddy image"
  debug:
    msg: "Extracting static assets from the Airflow image to build the Caddy reverse proxy."
  when: not fast_deploy | default(false)

- name: Prepare Caddy asset extraction directory
  file:
    path: "{{ airflow_master_dir }}/caddy_build_assets"
    state: "{{ item }}"
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    mode: '0755'
  loop:
    - absent
    - directory
  become: yes
  when: not fast_deploy | default(false)

- name: Ensure subdirectories exist with correct permissions
  file:
    path: "{{ airflow_master_dir }}/caddy_build_assets/{{ item }}"
    state: directory
    owner: "{{ ssh_user }}"
    group: "{{ deploy_group }}"
    mode: '0755'
  loop:
    - "appbuilder"
    - "dist"
  become: yes
  when: not fast_deploy | default(false)

- name: Extract static assets from Airflow image for Caddy build
  shell: |
    set -e
    CONTAINER_ID=$(docker create {{ airflow_image_name }})
    # Dynamically find paths inside the container
    APPBUILDER_PATH=$(docker run --rm --entrypoint "" {{ airflow_image_name }} python -c 'import os, flask_appbuilder; print(os.path.join(os.path.dirname(flask_appbuilder.__file__), "static", "appbuilder"))')
    AIRFLOW_DIST_PATH=$(docker run --rm --entrypoint "" {{ airflow_image_name }} python -c 'import os, airflow; print(os.path.join(os.path.dirname(airflow.__file__), "www/static/dist"))')
    # Copy assets from container to host
    docker cp "${CONTAINER_ID}:${APPBUILDER_PATH}/." "./caddy_build_assets/appbuilder"
    docker cp "${CONTAINER_ID}:${AIRFLOW_DIST_PATH}/." "./caddy_build_assets/dist"
    docker rm -f $CONTAINER_ID
    # Pre-compress assets
    find ./caddy_build_assets/appbuilder -type f -print0 | xargs -0 gzip -k -9
    find ./caddy_build_assets/dist -type f -print0 | xargs -0 gzip -k -9
  args:
    chdir: "{{ airflow_master_dir }}"
    executable: /bin/bash
  become: yes
  register: asset_extraction
  changed_when: asset_extraction.rc == 0
  when: not fast_deploy | default(false)

- name: "Log: Building Caddy reverse proxy image"
  debug:
    msg: "Building the Caddy image (pangramia/ytdlp-ops-caddy:latest) to serve static assets."

- name: Build Caddy image
  community.docker.docker_image:
    name: "pangramia/ytdlp-ops-caddy:latest"
    build:
      path: "{{ airflow_master_dir }}"
      dockerfile: "Dockerfile.caddy"
    source: build
    force_source: true
  when: not fast_deploy | default(false)

- name: "Log: Starting Airflow services"
  debug:
    msg: "Starting Airflow core services (webserver, scheduler, etc.) on the master node using docker-compose."

- name: Start Airflow master service
  community.docker.docker_compose_v2:
    project_src: "{{ airflow_master_dir }}"
    files:
      - "configs/docker-compose-master.yaml"
      - "configs/docker-compose-ytdlp-ops.yaml"
    state: present
    remove_orphans: true
    pull: "{{ 'never' if fast_deploy | default(false) else 'missing' }}"