131 lines
4.1 KiB
YAML
131 lines
4.1 KiB
YAML
---
|
|
- name: Deploy Airflow DL Worker Stack
|
|
hosts: airflow_workers
|
|
vars_files:
|
|
- "{{ inventory_dir }}/group_vars/all/vault.yml"
|
|
- "{{ inventory_dir }}/group_vars/all/generated_vars.yml"
|
|
pre_tasks:
|
|
- name: Announce fast deploy mode if enabled
|
|
debug:
|
|
msg: "🚀 FAST DEPLOY MODE ENABLED: Skipping Docker image builds and pulls. 🚀"
|
|
when: fast_deploy | default(false)
|
|
run_once: true
|
|
|
|
- name: Install python3-pip
|
|
ansible.builtin.apt:
|
|
name: python3-pip
|
|
state: present
|
|
become: yes
|
|
|
|
- name: Install required python packages for ytops-client on host
|
|
ansible.builtin.pip:
|
|
name:
|
|
- thrift
|
|
- aria2p
|
|
- PyYAML
|
|
state: present
|
|
extra_args: --break-system-packages
|
|
become: yes
|
|
tasks:
|
|
|
|
- name: Ensure worker directory exists
|
|
file:
|
|
path: "{{ airflow_worker_dir }}"
|
|
state: directory
|
|
owner: "{{ ansible_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0755'
|
|
become: yes
|
|
|
|
- name: Template .env file for worker
|
|
template:
|
|
src: templates/.env.j2
|
|
dest: "{{ airflow_worker_dir }}/.env"
|
|
mode: '0600'
|
|
vars:
|
|
service_role: "worker"
|
|
|
|
- name: Template docker-compose file for Airflow worker
|
|
template:
|
|
src: ../airflow/configs/docker-compose-dl.yaml.j2
|
|
dest: "{{ airflow_worker_dir }}/configs/docker-compose-dl.yaml"
|
|
mode: '0644'
|
|
|
|
- name: Ensure configs directory exists for config generator
|
|
file:
|
|
path: "{{ airflow_worker_dir }}/configs"
|
|
state: directory
|
|
owner: "{{ ansible_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0755'
|
|
become: yes
|
|
|
|
- name: Sync config generator script
|
|
ansible.posix.synchronize:
|
|
src: ../airflow/generate_envoy_config.py
|
|
dest: "{{ airflow_worker_dir }}/"
|
|
archive: yes
|
|
rsync_path: "sudo rsync"
|
|
|
|
- name: Sync config generator templates
|
|
ansible.posix.synchronize:
|
|
src: ../airflow/configs/{{ item }}
|
|
dest: "{{ airflow_worker_dir }}/configs/"
|
|
archive: yes
|
|
rsync_path: "sudo rsync"
|
|
loop:
|
|
- docker-compose.config-generate.yaml
|
|
- envoy.yaml.j2
|
|
- docker-compose.camoufox.yaml.j2
|
|
- docker-compose-ytdlp-ops.yaml.j2
|
|
|
|
- name: Build Airflow worker image from local Dockerfile
|
|
community.docker.docker_image:
|
|
name: "{{ airflow_image_name }}"
|
|
build:
|
|
path: "{{ airflow_worker_dir }}"
|
|
dockerfile: "Dockerfile"
|
|
source: build
|
|
force_source: true
|
|
when: not fast_deploy | default(false)
|
|
|
|
- name: Pull ytdlp-ops-server image only
|
|
community.docker.docker_image:
|
|
name: "{{ ytdlp_ops_image }}"
|
|
source: pull
|
|
when: not fast_deploy | default(false)
|
|
|
|
- name: Generate dynamic configs (camoufox + envoy)
|
|
shell:
|
|
cmd: "docker compose --project-directory . -f configs/docker-compose.config-generate.yaml run --rm config-generator"
|
|
chdir: "{{ airflow_worker_dir }}"
|
|
|
|
- name: Start worker services
|
|
community.docker.docker_compose_v2:
|
|
project_src: "{{ airflow_worker_dir }}"
|
|
files:
|
|
- configs/docker-compose-dl.yaml
|
|
- configs/docker-compose-ytdlp-ops.yaml
|
|
state: present
|
|
remove_orphans: true
|
|
pull: "{{ 'never' if fast_deploy | default(false) else 'missing' }}"
|
|
|
|
- name: Stop camoufox services (deprecated)
|
|
community.docker.docker_compose_v2:
|
|
project_src: "{{ airflow_worker_dir }}"
|
|
files:
|
|
- configs/docker-compose.camoufox.yaml
|
|
state: absent
|
|
ignore_errors: true
|
|
|
|
- name: Update Airflow variable with S3 worker hostnames
|
|
ansible.builtin.command: >
|
|
docker compose exec -T airflow-scheduler
|
|
airflow variables set s3_worker_hostnames
|
|
'{{ groups["airflow_workers"] | map("regex_replace", "\\..*", "") | list | to_json }}'
|
|
args:
|
|
chdir: "{{ airflow_master_dir }}"
|
|
become: yes
|
|
delegate_to: "{{ groups['airflow_master'][0] }}"
|
|
run_once: true
|