--- - name: Deploy Airflow Master hosts: airflow_master gather_facts: yes vars_files: - "{{ inventory_dir }}/group_vars/all/generated_vars.yml" - "{{ inventory_dir }}/group_vars/all/vault.yml" vars: envoy_port: 9980 envoy_admin_port: 9981 pre_tasks: - name: Announce master deployment debug: msg: "Starting deployment for Airflow Master: {{ inventory_hostname }} ({{ ansible_host }})" - name: Configure Redis memory overcommit setting copy: src: "configs/etc/sysctl.d/99-redis-overcommit.conf" dest: "/etc/sysctl.d/99-redis-overcommit.conf" owner: root group: root mode: '0644' become: yes register: redis_sysctl_config_copy - name: Configure system limits copy: src: "configs/etc/sysctl.d/99-system-limits.conf" dest: "/etc/sysctl.d/99-system-limits.conf" owner: root group: root mode: '0644' become: yes register: limits_sysctl_config_copy - name: Apply sysctl settings for Redis command: sysctl --system become: yes when: redis_sysctl_config_copy.changed - name: Apply sysctl settings for system limits command: sysctl --system become: yes when: limits_sysctl_config_copy.changed - name: Configure system timezone # Ensures all services and logs on this node use a consistent timezone. community.general.timezone: name: "{{ host_timezone }}" become: yes - name: Install NTP for time synchronization ansible.builtin.apt: name: ntp state: present become: yes - name: Ensure NTP service is started and enabled ansible.builtin.service: name: ntp state: started enabled: yes become: yes - name: Set deploy_group to a valid single group name set_fact: deploy_group: "ytdl" - name: Ensure deploy group exists group: name: "{{ deploy_group }}" state: present become: yes - name: Ensure deploy user exists user: name: "{{ ansible_user }}" group: "{{ deploy_group }}" state: present become: yes - name: Validate deploy_group variable ansible.builtin.assert: that: - deploy_group is defined - deploy_group is string - "',' not in deploy_group" - "' ' not in deploy_group" fail_msg: "The 'deploy_group' variable ('{{ deploy_group }}') must be a single, valid group name. It should not contain commas or spaces." - name: Check for swapfile stat: path: /swapfile register: swap_file become: yes - name: Create 8GB swapfile command: fallocate -l 8G /swapfile when: not swap_file.stat.exists become: yes - name: Set swapfile permissions file: path: /swapfile mode: '0600' when: not swap_file.stat.exists become: yes - name: Make swap command: mkswap /swapfile when: not swap_file.stat.exists become: yes - name: Check current swap status command: swapon --show register: swap_status changed_when: false become: yes - name: Enable swap command: swapon /swapfile when: "'/swapfile' not in swap_status.stdout" become: yes - name: Add swapfile to fstab lineinfile: path: /etc/fstab regexp: '^/swapfile' line: '/swapfile none swap sw 0 0' state: present become: yes - name: Get GID of the deploy group getent: database: group key: "{{ deploy_group }}" register: deploy_group_info become: yes - name: Set deploy_group_gid fact set_fact: deploy_group_gid: "{{ deploy_group_info.ansible_facts.getent_group[deploy_group][1] }}" when: deploy_group_info.ansible_facts.getent_group is defined and deploy_group in deploy_group_info.ansible_facts.getent_group - name: Ensure deploy_group_gid is set to a valid value set_fact: deploy_group_gid: "0" when: deploy_group_gid is not defined or deploy_group_gid == "" - name: Ensure master directory exists ansible.builtin.file: path: "{{ airflow_master_dir }}" state: directory owner: "{{ ansible_user }}" group: "{{ deploy_group }}" mode: '0755' become: yes - name: Sync python packages to master for build context ansible.posix.synchronize: src: "../{{ item }}/" dest: "{{ airflow_master_dir }}/{{ item }}/" rsync_opts: - "--delete" - "--exclude=.DS_Store" - "--exclude=__pycache__" - "--exclude='*.pyc'" recursive: yes perms: yes loop: - "thrift_model" - "pangramia" - "ytops_client" - "yt_ops_services" become: yes become_user: "{{ ansible_user }}" - name: Ensure bin directory exists on master for build context ansible.builtin.file: path: "{{ airflow_master_dir }}/bin" state: directory mode: '0755' become: yes become_user: "{{ ansible_user }}" - name: Sync root files and client utilities to master for build context ansible.posix.synchronize: src: "../{{ item }}" dest: "{{ airflow_master_dir }}/{{ item }}" perms: yes loop: - "setup.py" - "VERSION" - "bin/ytops-client" become: yes become_user: "{{ ansible_user }}" - name: Ensure Airflow project directory is writable by the container user (UID 50000) ansible.builtin.file: path: "{{ airflow_master_dir }}" owner: 50000 group: 50000 become: yes - name: Ensure Airflow subdirectories are writable by the container user (UID 50000) ansible.builtin.file: path: "{{ item }}" owner: 50000 group: 50000 recurse: yes state: directory loop: - "{{ airflow_master_dir }}/dags" - "{{ airflow_master_dir }}/logs" - "{{ airflow_master_dir }}/plugins" - "{{ airflow_master_dir }}/config" become: yes tasks: - name: Install pipx ansible.builtin.apt: name: pipx state: present become: yes - name: Install Glances for system monitoring ansible.builtin.command: pipx install glances[all] args: creates: "{{ ansible_env.HOME }}/.local/bin/glances" become: yes become_user: "{{ ansible_user }}" # Include Docker health check - name: Include Docker health check tasks include_tasks: tasks/docker_health_check.yml - name: Generate Docker Compose configurations ansible.builtin.command: > docker compose --project-directory . -f configs/docker-compose.config-generate.yaml run --rm config-generator args: chdir: "{{ airflow_master_dir }}" become: yes become_user: "{{ ansible_user }}" register: config_generator_result changed_when: "'Creating' in config_generator_result.stdout or 'Recreating' in config_generator_result.stdout" - name: Show config generator output ansible.builtin.debug: var: config_generator_result.stdout_lines when: config_generator_result.changed roles: - ytdlp-master - airflow-master post_tasks: - name: Include camoufox verification tasks include_tasks: tasks/verify_camoufox.yml when: not fast_deploy | default(false) - name: Run regression test command: > docker exec -i airflow-regression-runner python3 /opt/airflow/dags/scripts/regression.py --client "{{ regression_client | default('mweb') }}" --workers {{ regression_workers | default(4) }} --workers-per-bunch {{ regression_workers_per_bunch | default(4) }} --run-time-min {{ regression_run_time_min | default(120) }} --input-file "{{ regression_input_file | default('/opt/airflow/inputfiles/video_ids.csv') }}" --progress-interval-min {{ regression_progress_interval_min | default(2) }} --report-file "{{ regression_report_file | default('/opt/airflow/downloadfiles/regression_report.csv') }}" {% if regression_cleanup | default(true) %}--cleanup{% endif %} register: regression_test_result changed_when: false when: run_regression_test | default(false) - name: Display regression test output debug: var: regression_test_result.stdout_lines when: run_regression_test | default(false)