281 lines
8.6 KiB
YAML
281 lines
8.6 KiB
YAML
---
|
|
- name: Ensure worker is not paused on deploy (remove .lock file)
|
|
file:
|
|
path: "{{ airflow_worker_dir }}/inputfiles/AIRFLOW.PREVENT_URL_PULL.lockfile"
|
|
state: absent
|
|
become: yes
|
|
|
|
- name: Clean up old renamed lock files (older than 7 days)
|
|
ansible.builtin.find:
|
|
paths: "{{ airflow_worker_dir }}/inputfiles"
|
|
patterns: "AIRFLOW.PREVENT_URL_PULL.lockfile.removed-*"
|
|
age: "7d"
|
|
use_regex: false
|
|
register: old_lock_files
|
|
become: yes
|
|
|
|
- name: Remove found old lock files
|
|
ansible.builtin.file:
|
|
path: "{{ item.path }}"
|
|
state: absent
|
|
loop: "{{ old_lock_files.files }}"
|
|
become: yes
|
|
when: old_lock_files.files | length > 0
|
|
|
|
- name: Ensure YT-DLP worker inputfiles directory exists
|
|
file:
|
|
path: "{{ airflow_worker_dir }}/inputfiles"
|
|
state: directory
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0755'
|
|
become: yes
|
|
|
|
- name: Ensure YT-DLP worker logs directory exists
|
|
file:
|
|
path: "{{ airflow_worker_dir }}/logs"
|
|
state: directory
|
|
owner: "{{ airflow_uid }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0775'
|
|
become: yes
|
|
|
|
- name: Check if YT-DLP worker deployment directory exists
|
|
stat:
|
|
path: "{{ airflow_worker_dir }}"
|
|
register: worker_dir_stat
|
|
|
|
- name: Ensure YT-DLP worker deployment directory exists
|
|
file:
|
|
path: "{{ airflow_worker_dir }}"
|
|
state: directory
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0755'
|
|
become: yes
|
|
when: not worker_dir_stat.stat.exists
|
|
|
|
- name: Ensure YT-DLP worker configs directory exists
|
|
file:
|
|
path: "{{ airflow_worker_dir }}/configs"
|
|
state: directory
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0755'
|
|
become: yes
|
|
|
|
- name: "Log: Syncing YT-DLP service files"
|
|
debug:
|
|
msg: "Syncing YT-DLP service components (config generator, envoy templates) to the worker node."
|
|
|
|
- name: Sync YT-DLP config generator to worker
|
|
synchronize:
|
|
src: "../airflow/generate_envoy_config.py"
|
|
dest: "{{ airflow_worker_dir }}/"
|
|
archive: yes
|
|
rsync_path: "sudo rsync"
|
|
rsync_opts: "{{ rsync_default_opts }}"
|
|
|
|
- name: Sync YT-DLP config files to worker
|
|
synchronize:
|
|
src: "../airflow/configs/{{ item }}"
|
|
dest: "{{ airflow_worker_dir }}/configs/"
|
|
archive: yes
|
|
recursive: yes
|
|
rsync_path: "sudo rsync"
|
|
rsync_opts: "{{ rsync_default_opts }}"
|
|
loop:
|
|
- "docker-compose-ytdlp-ops.yaml.j2"
|
|
- "docker-compose.config-generate.yaml"
|
|
- "envoy.yaml.j2"
|
|
|
|
- name: Sync Airflow build context to worker
|
|
synchronize:
|
|
src: "../{{ item }}"
|
|
dest: "{{ airflow_worker_dir }}/"
|
|
archive: yes
|
|
recursive: yes
|
|
rsync_path: "sudo rsync"
|
|
rsync_opts: "{{ rsync_default_opts }}"
|
|
loop:
|
|
- "airflow/Dockerfile"
|
|
- "setup.py"
|
|
- "VERSION"
|
|
- "yt_ops_services"
|
|
- "thrift_model"
|
|
- "pangramia"
|
|
|
|
- name: Create .env file for YT-DLP worker service
|
|
template:
|
|
src: "../../templates/.env.j2"
|
|
dest: "{{ airflow_worker_dir }}/.env"
|
|
mode: "{{ file_permissions }}"
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
become: yes
|
|
vars:
|
|
service_role: "worker"
|
|
server_identity: "ytdlp-ops-service-worker-{{ inventory_hostname }}"
|
|
|
|
- name: Create symlink for .env in configs directory for manual docker-compose commands
|
|
file:
|
|
src: "../.env"
|
|
dest: "{{ airflow_worker_dir }}/configs/.env"
|
|
state: link
|
|
force: yes
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
become: yes
|
|
|
|
- name: Log in to Docker Hub to pull private images
|
|
community.docker.docker_login:
|
|
username: "{{ dockerhub_user }}"
|
|
password: "{{ vault_dockerhub_password }}"
|
|
when: vault_dockerhub_password is defined and vault_dockerhub_password | length > 0
|
|
|
|
- name: "Log: Generating YT-DLP service configurations"
|
|
debug:
|
|
msg: "Running the configuration generator script inside a temporary Docker container. This creates docker-compose, envoy, and camoufox files based on .env variables."
|
|
|
|
- name: Ensure previously generated config files are removed before generation
|
|
file:
|
|
path: "{{ item }}"
|
|
state: absent
|
|
loop:
|
|
- "{{ airflow_worker_dir }}/envoy.yaml"
|
|
- "{{ airflow_worker_dir }}/configs/docker-compose.camoufox.yaml"
|
|
- "{{ airflow_worker_dir }}/configs/camoufox_endpoints.json"
|
|
become: yes
|
|
|
|
- name: Create placeholder envoy.yaml to prevent Docker from creating a directory
|
|
file:
|
|
path: "{{ airflow_worker_dir }}/envoy.yaml"
|
|
state: touch
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0664'
|
|
become: yes
|
|
|
|
- name: Generate YT-DLP service configurations
|
|
shell:
|
|
cmd: "docker compose --project-directory {{ airflow_worker_dir }} -f configs/docker-compose.config-generate.yaml run --rm config-generator"
|
|
chdir: "{{ airflow_worker_dir }}"
|
|
become: yes
|
|
become_user: "{{ ssh_user }}"
|
|
|
|
- name: Clean up old root docker-compose files to prevent conflicts
|
|
ansible.builtin.file:
|
|
path: "{{ airflow_worker_dir }}/{{ item }}"
|
|
state: absent
|
|
loop:
|
|
- "docker-compose.yml"
|
|
- "docker-compose.yaml"
|
|
- "docker-compose.override.yml"
|
|
- "docker-compose.airflow.yml"
|
|
become: yes
|
|
|
|
- name: Template docker-compose file for Airflow worker
|
|
template:
|
|
src: "{{ playbook_dir }}/../airflow/configs/docker-compose-dl.yaml.j2"
|
|
dest: "{{ airflow_worker_dir }}/configs/docker-compose.airflow.yml"
|
|
mode: "{{ file_permissions }}"
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
become: yes
|
|
|
|
- name: "Log: Building Airflow image"
|
|
debug:
|
|
msg: "Building the Airflow image locally. This image contains all dependencies for running DAGs."
|
|
|
|
- name: Build Airflow image from local Dockerfile
|
|
community.docker.docker_image:
|
|
name: "pangramia/ytdlp-ops-airflow:latest"
|
|
build:
|
|
path: "{{ airflow_worker_dir }}"
|
|
dockerfile: "Dockerfile"
|
|
source: build
|
|
force_source: true
|
|
when: not fast_deploy | default(false)
|
|
|
|
- name: "Log: Building aria2-pro image"
|
|
debug:
|
|
msg: "Building the aria2-pro image locally. This image provides the download manager."
|
|
when: not fast_deploy | default(false)
|
|
|
|
- name: Build aria2-pro image from docker-compose
|
|
ansible.builtin.command: >
|
|
docker compose -f configs/docker-compose.airflow.yml build aria2-pro
|
|
args:
|
|
chdir: "{{ airflow_worker_dir }}"
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
register: docker_build_result
|
|
changed_when: "'Building' in docker_build_result.stdout or 'writing image' in docker_build_result.stdout"
|
|
when: not fast_deploy | default(false)
|
|
|
|
# - name: "Log: Building Camoufox (remote browser) image"
|
|
# debug:
|
|
# msg: "Building the Camoufox image locally. This image provides remote-controlled Firefox browsers for token generation."
|
|
#
|
|
# - name: Build Camoufox image from local Dockerfile
|
|
# community.docker.docker_image:
|
|
# name: "camoufox:latest"
|
|
# build:
|
|
# path: "{{ airflow_worker_dir }}/camoufox"
|
|
# source: build
|
|
# force_source: true
|
|
# when: not fast_deploy | default(false)
|
|
|
|
- name: Ensure correct permissions for build context after generation
|
|
file:
|
|
path: "{{ airflow_worker_dir }}"
|
|
state: directory
|
|
owner: "{{ ssh_user }}"
|
|
group: "{{ deploy_group }}"
|
|
recurse: yes
|
|
become: yes
|
|
|
|
- name: Check for shadowsocks-rust proxy compose file
|
|
stat:
|
|
path: "/srv/shadowsocks-rust/docker-compose.proxies.yaml"
|
|
register: proxy_compose_file
|
|
|
|
- name: "Log: Stopping worker services before start"
|
|
debug:
|
|
msg: "Stopping all worker services to ensure a clean start."
|
|
|
|
- name: Stop all worker services
|
|
community.docker.docker_compose_v2:
|
|
project_src: "{{ airflow_worker_dir }}"
|
|
files:
|
|
- "configs/docker-compose-ytdlp-ops.yaml"
|
|
- "configs/docker-compose.airflow.yml"
|
|
state: absent
|
|
remove_volumes: true # Corresponds to docker compose down -v
|
|
|
|
- name: Forcefully remove project-specific Docker volumes to fix corruption issues
|
|
ansible.builtin.shell: "docker volume ls -q --filter 'label=com.docker.compose.project=ytdlp-ops-worker' | xargs -r docker volume rm --force"
|
|
become: yes
|
|
register: removed_volumes
|
|
changed_when: removed_volumes.stdout | length > 0
|
|
failed_when: false
|
|
|
|
- name: "Log: Starting all worker services"
|
|
debug:
|
|
msg: "Starting all worker services: ytdlp-ops, and airflow-worker."
|
|
|
|
- name: Start all worker services
|
|
community.docker.docker_compose_v2:
|
|
project_src: "{{ airflow_worker_dir }}"
|
|
files:
|
|
- "configs/docker-compose-ytdlp-ops.yaml"
|
|
- "configs/docker-compose.airflow.yml"
|
|
state: present
|
|
remove_orphans: true
|
|
pull: "{{ 'never' if fast_deploy | default(false) else 'missing' }}"
|
|
recreate: always # Corresponds to --force-recreate
|
|
|
|
# - name: Include camoufox verification tasks
|
|
# include_tasks: ../../../tasks/verify_camoufox.yml
|
|
# when: not fast_deploy | default(false)
|