yt-dlp-dags/ansible/playbook-full.yml

---
- name: Deploy entire Airflow cluster
  hosts: all
  gather_facts: true
  vars_files:
    - "{{ inventory_dir }}/group_vars/all/generated_vars.yml"
    - "{{ inventory_dir }}/group_vars/all/vault.yml"
  pre_tasks:
    - name: Announce fast deploy mode if enabled
      debug:
        msg: "🚀 FAST DEPLOY MODE ENABLED: Skipping Docker image builds and pulls. 🚀"
      when: fast_deploy | default(false)
      run_once: true

    - name: Check if Docker is already installed
      ansible.builtin.stat:
        path: /usr/bin/docker
      register: docker_binary

    - name: Install Docker if not present
      block:
        - name: Add Docker's official GPG key
          ansible.builtin.apt_key:
            url: https://download.docker.com/linux/ubuntu/gpg
            state: present

        - name: Find and remove any existing Docker repository files to avoid conflicts
          block:
            - name: Find legacy docker repository files
              ansible.builtin.find:
                paths: /etc/apt/sources.list.d/
                patterns: '*.list'
                contains: 'deb .*download.docker.com'
              register: legacy_docker_repo_files

            - name: Remove legacy docker repository files
              ansible.builtin.file:
                path: "{{ item.path }}"
                state: absent
              loop: "{{ legacy_docker_repo_files.files }}"

        - name: Set up the Docker repository
          ansible.builtin.apt_repository:
            repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename }} stable"
            state: present

        - name: Install prerequisites for Docker
          ansible.builtin.apt:
            name:
              - apt-transport-https
              - ca-certificates
              - curl
              - software-properties-common
              - vim
              - python3-pip
              - iputils-ping
              - traceroute
              - fail2ban
              - conntrack
              - tcpdump
            state: present
            update_cache: yes

        - name: Install Docker Engine and Docker Compose
          ansible.builtin.apt:
            name:
              - docker-ce
              - docker-ce-cli
              - containerd.io
              - docker-compose-plugin
              - python3-docker
            state: present
            update_cache: yes
      when: not docker_binary.stat.exists
      become: yes
  tasks:

    - name: Install pipx
      ansible.builtin.apt:
        name: pipx
        state: present
      become: yes

    - name: Install Glances for system monitoring
      ansible.builtin.command: pipx install glances[all]
      args:
        creates: "{{ ansible_env.HOME }}/.local/bin/glances"
      become: yes
      become_user: "{{ ansible_user }}"

    - name: Ensure Docker service is started and enabled
      ansible.builtin.service:
        name: docker
        state: started
        enabled: yes
      become: yes

    - name: Add deploy user to the docker group
      ansible.builtin.user:
        name: "{{ ansible_user }}"
        groups: docker
        append: yes
      become: yes

    - name: Reset SSH connection to apply group changes
      ansible.builtin.meta: reset_connection


    - name: Ensure shared Docker network exists
      community.docker.docker_network:
        name: airflow_proxynet
        driver: bridge


- name: Deploy master
  import_playbook: playbook-master.yml
  when: inventory_hostname in groups['airflow_master']

- name: Deploy workers
  import_playbook: playbook-worker.yml
  when: inventory_hostname in groups['airflow_workers']

- name: Deploy and Reload Airflow Task Hook
  hosts: all
  gather_facts: no
  vars_files:
    - "{{ inventory_dir }}/group_vars/all/generated_vars.yml"
    - "{{ inventory_dir }}/group_vars/all/vault.yml"
  tasks:
    - name: Ensure config directory exists on MASTER server
      when: inventory_hostname in groups['airflow_master']
      ansible.builtin.file:
        path: "{{ airflow_master_dir }}/config"
        state: directory
        owner: "{{ ansible_user }}"
        group: "{{ ansible_user }}"
        mode: '0755'
      become: yes

    - name: Ensure config directory exists on WORKER server
      when: inventory_hostname in groups['airflow_workers']
      ansible.builtin.file:
        path: "{{ airflow_worker_dir }}/config"
        state: directory
        owner: "{{ ansible_user }}"
        group: "{{ ansible_user }}"
        mode: '0755'
      become: yes

    - name: Sync custom_task_hooks.py to MASTER server
      when: inventory_hostname in groups['airflow_master']
      synchronize:
        src: "../airflow/config/custom_task_hooks.py"
        dest: "{{ airflow_master_dir }}/config/"
        archive: yes
        rsync_path: "sudo rsync"

    - name: Sync airflow_local_settings.py to MASTER server
      when: inventory_hostname in groups['airflow_master']
      synchronize:
        src: "../airflow/config/airflow_local_settings.py"
        dest: "{{ airflow_master_dir }}/config/"
        archive: yes
        rsync_path: "sudo rsync"

    - name: Sync custom_task_hooks.py to WORKER server
      when: inventory_hostname in groups['airflow_workers']
      synchronize:
        src: "../airflow/config/custom_task_hooks.py"
        dest: "{{ airflow_worker_dir }}/config/"
        archive: yes
        rsync_path: "sudo rsync"

    - name: Sync airflow_local_settings.py to WORKER server
      when: inventory_hostname in groups['airflow_workers']
      synchronize:
        src: "../airflow/config/airflow_local_settings.py"
        dest: "{{ airflow_worker_dir }}/config/"
        archive: yes
        rsync_path: "sudo rsync"

    - name: Restart Airflow services on MASTER to apply hook
      when: inventory_hostname in groups['airflow_master']
      ansible.builtin.command:
        cmd: "docker compose restart airflow-scheduler airflow-webserver airflow-master-worker airflow-triggerer"
        chdir: "{{ airflow_master_dir }}"
      become: yes

    - name: Restart Airflow worker on WORKER to apply hook
      when: inventory_hostname in groups['airflow_workers']
      ansible.builtin.command:
        cmd: "docker compose restart airflow-worker-dl airflow-worker-auth airflow-worker-mgmt airflow-worker-s3"
        chdir: "{{ airflow_worker_dir }}"
      become: yes

    - name: Update Airflow variable with S3 worker hostnames
      ansible.builtin.command: >
        docker compose exec -T airflow-scheduler
        airflow variables set s3_worker_hostnames
        '{{ groups["airflow_workers"] | map("regex_replace", "\\..*", "") | list | to_json }}'
      args:
        chdir: "{{ airflow_master_dir }}"
      become: yes
      when: inventory_hostname in groups['airflow_master']
      run_once: true