yt-dlp-dags/ansible/playbook-docker-services-setup.yml

---
# This playbook orchestrates the entire setup for the stress test environment.
# It is composed of multiple plays and imported playbooks to ensure modularity and correct execution order.

# -------------------------------------------------------------------------------------------------
# PHASE 1: Base System Configuration
# Ensures all nodes have the necessary base packages, user configurations, and Docker installed.
# -------------------------------------------------------------------------------------------------
- name: "PHASE 1.1: Import base system setup playbook"
  import_playbook: playbook-base-system.yml

- name: "PHASE 1.2: Import proxy deployment playbook"
  import_playbook: playbook-proxies.yml

# -------------------------------------------------------------------------------------------------
# PHASE 2: Application Directory and Code Setup
# Creates the necessary directory structure and syncs the application code.
# -------------------------------------------------------------------------------------------------
- name: "PHASE 2.1: Create Base Directories"
  hosts: all
  gather_facts: no
  vars_files:
    - "group_vars/all/generated_vars.stress.yml"
    - "group_vars/all/vault.yml"
  tasks:
    - name: Define base directory for node
      ansible.builtin.set_fact:
        base_dir: "{{ airflow_master_dir if inventory_hostname in groups['airflow_master'] else airflow_worker_dir }}"

    - name: Ensure base directories and subdirectories exist
      ansible.builtin.file:
        path: "{{ base_dir }}/{{ item }}"
        state: directory
        owner: "{{ ansible_user }}"
        group: "{{ deploy_group }}"
        mode: '0755'
        recurse: yes
      loop:
        - "" # for the base directory itself
        - "inputfiles"
        - "run/docker_mount/fetched_info_jsons"
      become: yes

- name: "PHASE 2.2: Import playbook to install Python dependencies"
  import_playbook: playbook-stress-install-deps.yml

- name: "PHASE 2.3: Import playbook to sync local code"
  import_playbook: playbook-stress-sync-code.yml

# -------------------------------------------------------------------------------------------------
# PHASE 3: Environment and Service Configuration
# Generates the .env file and starts the role-specific services on master and workers.
# -------------------------------------------------------------------------------------------------
- name: "PHASE 3.1: Import playbook to generate .env file"
  import_playbook: playbook-stress-generate-env.yml

- name: "PHASE 3.2: Master Node Services Setup"
  hosts: airflow_master
  gather_facts: no
  vars_files:
    - "group_vars/all/generated_vars.stress.yml"
    - "group_vars/all/vault.yml"
  tasks:
    - name: Configure system performance and kernel settings
      ansible.builtin.copy:
        src: "configs/etc/sysctl.d/99-system-limits.conf"
        dest: "/etc/sysctl.d/99-system-limits.conf"
        owner: root
        group: root
        mode: '0644'
      become: yes
      register: sysctl_config_copy

    - name: Apply sysctl settings
      ansible.builtin.command: sysctl --system
      become: yes
      when: sysctl_config_copy.changed

    - name: Ensure MinIO data directory exists
      ansible.builtin.file:
        path: "{{ airflow_master_dir }}/minio-data"
        state: directory
        owner: "{{ ansible_user }}"
        group: "{{ deploy_group }}"
        mode: '0755'
      become: yes

    - name: Template Docker Compose file for master services
      ansible.builtin.template:
        src: templates/docker-compose.stress-master.j2
        dest: "{{ airflow_master_dir }}/docker-compose.stress.yml"
        owner: "{{ ansible_user }}"
        group: "{{ deploy_group }}"
        mode: '0644'
      become: yes

    - name: Start master services (Redis, MinIO)
      community.docker.docker_compose_v2:
        project_src: "{{ airflow_master_dir }}"
        files:
          - docker-compose.stress.yml
        state: present
        remove_orphans: true
      become: yes

    - name: Wait for MinIO service to be ready
      ansible.builtin.wait_for:
        host: "{{ hostvars[inventory_hostname].ansible_host }}"
        port: 9000
        delay: 5
        timeout: 60
      delegate_to: localhost

    - name: Download MinIO Client (mc) if not present
      ansible.builtin.command:
        cmd: wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc
        creates: /usr/local/bin/mc
      become: yes

    - name: Ensure MinIO Client (mc) is executable
      ansible.builtin.file:
        path: /usr/local/bin/mc
        mode: '0755'
      become: yes

    - name: Configure mc alias for local MinIO
      ansible.builtin.command: >
        mc alias set local http://localhost:9000 {{ vault_s3_access_key_id }} {{ vault_s3_secret_access_key }}
      become: yes
      become_user: "{{ ansible_user }}"
      changed_when: false
      environment:
        HOME: "/home/{{ ansible_user }}"

    - name: Ensure S3 buckets exist in MinIO using mc
      ansible.builtin.command: >
        mc mb local/{{ item }}
      loop:
        - "stress-inputs"
        - "stress-jsons"
      become: yes
      become_user: "{{ ansible_user }}"
      register: mc_mb_result
      failed_when: >
        mc_mb_result.rc != 0 and
        "already exists" not in mc_mb_result.stderr
      changed_when: mc_mb_result.rc == 0
      environment:
        HOME: "/home/{{ ansible_user }}"

- name: "PHASE 3.3: Shared Storage Setup (s3fs)"
  hosts: airflow_master:airflow_workers
  gather_facts: no
  vars_files:
    - "group_vars/all/generated_vars.stress.yml"
    - "group_vars/all/vault.yml"
  tasks:
    - name: Define base directory for node
      ansible.builtin.set_fact:
        base_dir: "{{ airflow_master_dir if inventory_hostname in groups['airflow_master'] else airflow_worker_dir }}"

    - name: Mount S3 buckets via s3fs
      block:
        - name: Install s3fs for mounting S3 buckets
          ansible.builtin.apt:
            name: s3fs
            state: present
          become: yes

        - name: Configure s3fs credentials
          ansible.builtin.copy:
            content: "{{ vault_s3_access_key_id }}:{{ vault_s3_secret_access_key }}"
            dest: "/home/{{ ansible_user }}/.passwd-s3fs"
            owner: "{{ ansible_user }}"
            group: "{{ deploy_group }}"
            mode: '0600'
          become: yes

        - name: Mount S3 buckets for stress testing
          ansible.posix.mount:
            src: "s3fs#{{ item.bucket }}"
            path: "{{ item.path }}"
            fstype: fuse
            opts: "_netdev,allow_other,use_path_request_style,nonempty,url=http://{{ hostvars[groups['airflow_master'][0]].ansible_host }}:9000,passwd_file=/home/{{ ansible_user }}/.passwd-s3fs"
            state: mounted
          loop:
            - { bucket: 'stress-inputs', path: '{{ base_dir }}/inputfiles' }
            - { bucket: 'stress-jsons', path: '{{ base_dir }}/run/docker_mount/fetched_info_jsons' }
          become: yes