yt-dlp-dags/ansible/roles/airflow-master/tasks/main.yml

---
- name: Check if Airflow master deployment directory exists
  stat:
    path: "{{ airflow_master_dir }}"
  register: master_dir_stat

- name: Ensure Airflow master deployment directory exists
  file:
    path: "{{ airflow_master_dir }}"
    state: directory
    owner: "{{ ssh_user }}"
    group: ytdl
    mode: '0755'
  become: yes
  when: not master_dir_stat.stat.exists

- name: Check if source directories exist
  stat:
    path: "../{{ item }}"
  register: source_dirs
  loop:
    - "airflow/inputfiles"
    - "airflow/plugins"
    - "airflow/addfiles"
    - "airflow/bgutil-ytdlp-pot-provider"

- name: "Log: Syncing Airflow core files"
  debug:
    msg: "Syncing DAGs, configs, and Python source code to the master node."

- name: Sync Airflow master files
  synchronize:
    src: "../{{ item }}"
    dest: "{{ airflow_master_dir }}/"
    archive: yes
    recursive: yes
    rsync_path: "sudo rsync"
    rsync_opts: "{{ rsync_default_opts }}"
  loop:
    - "airflow/Dockerfile"
    - "airflow/.dockerignore"
    - "airflow/docker-compose-master.yaml"
    - "airflow/dags"
    - "airflow/config"
    - "yt_ops_package/setup.py"
    - "yt_ops_package/yt_ops_services"
    - "yt_ops_package/thrift_model"
    - "yt_ops_package/VERSION"
    - "yt_ops_package/pangramia"
    - "airflow/init-airflow.sh"
    - "airflow/update-yt-dlp.sh"
    - "airflow/nginx.conf"
    - "yt_ops_package/get_info_json_client.py"
    - "yt_ops_package/proxy_manager_client.py"
    - "token_generator"
    - "utils"

- name: Sync optional directories if they exist
  synchronize:
    src: "../{{ item.item }}/"
    dest: "{{ airflow_master_dir }}/{{ item.item | basename }}/"
    archive: yes
    recursive: yes
    delete: yes
    rsync_path: "sudo rsync"
    rsync_opts: "{{ rsync_default_opts }}"
  loop: "{{ source_dirs.results }}"
  when: item.stat.exists

- name: Sync pangramia thrift files
  synchronize:
    src: "../yt_ops_package/thrift_model/gen_py/pangramia/"
    dest: "{{ airflow_master_dir }}/pangramia/"
    archive: yes
    recursive: yes
    delete: yes
    rsync_path: "sudo rsync"
    rsync_opts: "{{ rsync_default_opts }}"

- name: Create .env file for Airflow master service
  template:
    src: "../../templates/.env.master.j2"
    dest: "{{ airflow_master_dir }}/.env"
    mode: "{{ file_permissions }}"
    owner: "{{ ssh_user }}"
    group: ytdl
  become: yes
  vars:
    service_role: "master"

- name: Template Minio connection file
  template:
    src: "../templates/minio_default_conn.json.j2"
    dest: "{{ airflow_master_dir }}/config/minio_default_conn.json"
    mode: "{{ file_permissions }}"
    owner: "{{ ssh_user }}"
    group: ytdl
  become: yes

- name: Template YT-DLP Redis connection file
  template:
    src: "../templates/ytdlp_redis_conn.json.j2"
    dest: "{{ airflow_master_dir }}/config/ytdlp_redis_conn.json"
    mode: "{{ file_permissions }}"
    owner: "{{ ssh_user }}"
    group: ytdl
  become: yes

- name: Create symlink for docker-compose.yaml
  file:
    src: "{{ airflow_master_dir }}/docker-compose-master.yaml"
    dest: "{{ airflow_master_dir }}/docker-compose.yaml"
    state: link
    owner: "{{ ssh_user }}"
    group: ytdl
    force: yes
    follow: no

- name: Ensure correct permissions for build context
  file:
    path: "{{ airflow_master_dir }}"
    state: directory
    owner: "{{ ssh_user }}"
    group: ytdl
    recurse: yes
  become: yes

- name: Verify Dockerfile exists in build directory
  stat:
    path: "{{ airflow_master_dir }}/Dockerfile"
  register: dockerfile_stat

- name: Fail if Dockerfile is missing
  fail:
    msg: "Dockerfile not found in {{ airflow_master_dir }}. Cannot build image."
  when: not dockerfile_stat.stat.exists

- name: "Log: Building Airflow Docker image"
  debug:
    msg: "Building the main Airflow Docker image ({{ airflow_image_name }}) locally on the master node. This may take a few minutes."

- name: Build Airflow master image
  community.docker.docker_image:
    name: "{{ airflow_image_name }}"
    build:
      path: "{{ airflow_master_dir }}"
      dockerfile: "Dockerfile"  # Explicitly specify the Dockerfile name
    source: build
    force_source: true

- name: Make Airflow init script executable
  file:
    path: "{{ airflow_master_dir }}/init-airflow.sh"
    mode: "0755"
  become: yes

- name: Run Airflow init script
  shell:
    cmd: "./init-airflow.sh"
    chdir: "{{ airflow_master_dir }}"
  become: yes
  become_user: "{{ ssh_user }}"

- name: "Log: Starting Airflow services"
  debug:
    msg: "Starting Airflow core services (webserver, scheduler, etc.) on the master node using docker-compose."

- name: Start Airflow master service
  community.docker.docker_compose_v2:
    project_src: "{{ airflow_master_dir }}"
    files:
      - "docker-compose-master.yaml"
    state: present
    remove_orphans: true