423 lines
12 KiB
YAML
423 lines
12 KiB
YAML
---
|
|
- name: Deploy Airflow Workers
|
|
hosts: airflow_workers
|
|
gather_facts: yes
|
|
vars_files:
|
|
- "{{ inventory_dir }}/group_vars/all/generated_vars.yml"
|
|
- "{{ inventory_dir }}/group_vars/all/vault.yml"
|
|
pre_tasks:
|
|
- name: Announce worker deployment
|
|
debug:
|
|
msg: "Starting deployment for Airflow Worker: {{ inventory_hostname }} ({{ ansible_user }}@{{ ansible_host }})"
|
|
|
|
- name: Configure system timezone
|
|
# Ensures all services and logs on this node use a consistent timezone.
|
|
community.general.timezone:
|
|
name: "{{ host_timezone }}"
|
|
become: yes
|
|
|
|
- name: Install NTP for time synchronization
|
|
ansible.builtin.apt:
|
|
name: ntp
|
|
state: present
|
|
become: yes
|
|
|
|
- name: Ensure NTP service is started and enabled
|
|
ansible.builtin.service:
|
|
name: ntp
|
|
state: started
|
|
enabled: yes
|
|
become: yes
|
|
|
|
- name: Set deploy_group to a valid single group name
|
|
set_fact:
|
|
deploy_group: "ytdl"
|
|
|
|
- name: Ensure deploy group exists
|
|
group:
|
|
name: "{{ deploy_group }}"
|
|
state: present
|
|
become: yes
|
|
|
|
- name: Ensure deploy user exists
|
|
user:
|
|
name: "{{ ansible_user }}"
|
|
group: "{{ deploy_group }}"
|
|
state: present
|
|
become: yes
|
|
|
|
- name: Validate deploy_group variable
|
|
ansible.builtin.assert:
|
|
that:
|
|
- deploy_group is defined
|
|
- deploy_group is string
|
|
- "',' not in deploy_group"
|
|
- "' ' not in deploy_group"
|
|
fail_msg: "The 'deploy_group' variable ('{{ deploy_group }}') must be a single, valid group name. It should not contain commas or spaces."
|
|
|
|
- name: Check for swapfile
|
|
stat:
|
|
path: /swapfile
|
|
register: swap_file
|
|
become: yes
|
|
|
|
- name: Create 8GB swapfile
|
|
command: fallocate -l 8G /swapfile
|
|
when: not swap_file.stat.exists
|
|
become: yes
|
|
|
|
- name: Set swapfile permissions
|
|
file:
|
|
path: /swapfile
|
|
mode: '0600'
|
|
when: not swap_file.stat.exists
|
|
become: yes
|
|
|
|
- name: Make swap
|
|
command: mkswap /swapfile
|
|
when: not swap_file.stat.exists
|
|
become: yes
|
|
|
|
- name: Check current swap status
|
|
command: swapon --show
|
|
register: swap_status
|
|
changed_when: false
|
|
become: yes
|
|
|
|
- name: Enable swap
|
|
command: swapon /swapfile
|
|
when: "'/swapfile' not in swap_status.stdout"
|
|
become: yes
|
|
|
|
- name: Add swapfile to fstab
|
|
lineinfile:
|
|
path: /etc/fstab
|
|
regexp: '^/swapfile'
|
|
line: '/swapfile none swap sw 0 0'
|
|
state: present
|
|
become: yes
|
|
|
|
- name: Get GID of the deploy group
|
|
getent:
|
|
database: group
|
|
key: "{{ deploy_group }}"
|
|
register: deploy_group_info
|
|
become: yes
|
|
|
|
- name: Set deploy_group_gid fact
|
|
set_fact:
|
|
deploy_group_gid: "{{ deploy_group_info.ansible_facts.getent_group[deploy_group][1] }}"
|
|
when: deploy_group_info.ansible_facts.getent_group is defined and deploy_group in deploy_group_info.ansible_facts.getent_group
|
|
|
|
- name: Ensure deploy_group_gid is set to a valid value
|
|
set_fact:
|
|
deploy_group_gid: "0"
|
|
when: deploy_group_gid is not defined or deploy_group_gid == ""
|
|
|
|
- name: Configure system limits
|
|
copy:
|
|
src: "configs/etc/sysctl.d/99-system-limits.conf"
|
|
dest: "/etc/sysctl.d/99-system-limits.conf"
|
|
owner: root
|
|
group: root
|
|
mode: '0644'
|
|
become: yes
|
|
register: limits_sysctl_config_copy
|
|
|
|
- name: Apply sysctl settings for system limits
|
|
command: sysctl --system
|
|
become: yes
|
|
when: limits_sysctl_config_copy.changed
|
|
|
|
- name: Create logs directory structure relative to deployment
|
|
file:
|
|
path: "./logs/yt-dlp-ops/communication_logs"
|
|
state: directory
|
|
mode: '0755'
|
|
owner: "{{ ansible_user }}"
|
|
group: "{{ deploy_group }}"
|
|
become: yes
|
|
|
|
- name: Ensure worker directory exists
|
|
ansible.builtin.file:
|
|
path: "{{ airflow_worker_dir }}"
|
|
state: directory
|
|
owner: "{{ ansible_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0755'
|
|
become: yes
|
|
|
|
- name: Ensure runtime data directories exist with correct ownership
|
|
ansible.builtin.file:
|
|
path: "{{ airflow_worker_dir }}/{{ item }}"
|
|
state: directory
|
|
owner: "{{ ansible_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0775'
|
|
recurse: yes
|
|
loop:
|
|
- "downloadfiles"
|
|
- "downloadfiles/videos"
|
|
- "downloadfiles/videos/in-progress"
|
|
- "downloadfiles/videos/ready"
|
|
- "inputfiles"
|
|
- "dumps"
|
|
become: yes
|
|
|
|
- name: Create .dockerignore on worker to exclude runtime data from build context
|
|
ansible.builtin.copy:
|
|
dest: "{{ airflow_worker_dir }}/.dockerignore"
|
|
content: |
|
|
# Exclude build artifacts and virtual environments
|
|
__pycache__/
|
|
*.pyc
|
|
*.pyo
|
|
.venv/
|
|
venv/
|
|
|
|
# Exclude sensitive information
|
|
.env
|
|
.vault_pass
|
|
|
|
# Exclude local development and OS-specific files
|
|
.DS_Store
|
|
.idea/
|
|
*.swp
|
|
|
|
# Exclude large directories with runtime data that should not be in the image
|
|
logs/
|
|
downloadfiles/
|
|
addfiles/
|
|
*downloads/
|
|
postgres-data/
|
|
redis-data/
|
|
minio-data/
|
|
owner: "{{ ansible_user }}"
|
|
group: "{{ deploy_group }}"
|
|
mode: '0644'
|
|
become: yes
|
|
|
|
- name: Sync python packages to worker for build context
|
|
ansible.posix.synchronize:
|
|
src: "../{{ item }}/"
|
|
dest: "{{ airflow_worker_dir }}/{{ item }}/"
|
|
rsync_opts:
|
|
- "--delete"
|
|
- "--exclude=.DS_Store"
|
|
- "--exclude=__pycache__"
|
|
- "--exclude='*.pyc'"
|
|
recursive: yes
|
|
perms: yes
|
|
loop:
|
|
- "thrift_model"
|
|
- "pangramia"
|
|
- "ytops_client"
|
|
- "yt_ops_services"
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
|
|
- name: Sync aria2-pro-docker to worker for build context
|
|
ansible.posix.synchronize:
|
|
src: "../airflow/aria2-pro-docker/"
|
|
dest: "{{ airflow_worker_dir }}/aria2-pro-docker/"
|
|
rsync_opts:
|
|
- "--delete"
|
|
recursive: yes
|
|
perms: yes
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
|
|
- name: Ensure bin directory exists on worker for build context
|
|
ansible.builtin.file:
|
|
path: "{{ airflow_worker_dir }}/bin"
|
|
state: directory
|
|
mode: '0755'
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
|
|
- name: Sync root files and client utilities to worker for build context
|
|
ansible.posix.synchronize:
|
|
src: "../{{ item }}"
|
|
dest: "{{ airflow_worker_dir }}/{{ item }}"
|
|
perms: yes
|
|
loop:
|
|
- "setup.py"
|
|
- "VERSION"
|
|
- "bin/ytops-client"
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
|
|
- name: Ensure Airflow project directory is writable by the container user (UID 50000)
|
|
ansible.builtin.file:
|
|
path: "{{ airflow_worker_dir }}"
|
|
owner: 50000
|
|
group: 50000
|
|
become: yes
|
|
|
|
- name: Ensure Airflow subdirectories are writable by the container user (UID 50000)
|
|
ansible.builtin.file:
|
|
path: "{{ item }}"
|
|
owner: 50000
|
|
group: 50000
|
|
recurse: yes
|
|
state: directory
|
|
loop:
|
|
- "{{ airflow_worker_dir }}/dags"
|
|
- "{{ airflow_worker_dir }}/logs"
|
|
- "{{ airflow_worker_dir }}/plugins"
|
|
- "{{ airflow_worker_dir }}/config"
|
|
become: yes
|
|
|
|
tasks:
|
|
- name: Install pipx
|
|
ansible.builtin.apt:
|
|
name: pipx
|
|
state: present
|
|
become: yes
|
|
|
|
- name: Install Glances for system monitoring
|
|
ansible.builtin.command: pipx install glances[all]
|
|
args:
|
|
creates: "{{ ansible_env.HOME }}/.local/bin/glances"
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
|
|
- name: Install base system packages for tools
|
|
ansible.builtin.apt:
|
|
name:
|
|
- unzip
|
|
- wget
|
|
- xz-utils
|
|
state: present
|
|
update_cache: yes
|
|
become: yes
|
|
|
|
- name: Install required Python packages
|
|
ansible.builtin.pip:
|
|
name:
|
|
- python-dotenv
|
|
- aria2p
|
|
- tabulate
|
|
- redis
|
|
- PyYAML
|
|
- aiothrift
|
|
- PySocks
|
|
state: present
|
|
extra_args: --break-system-packages
|
|
become: yes
|
|
|
|
- name: Install pinned Python packages
|
|
ansible.builtin.pip:
|
|
name:
|
|
- brotli==1.1.0
|
|
- certifi==2025.10.05
|
|
- curl-cffi==0.13.0
|
|
- mutagen==1.47.0
|
|
- pycryptodomex==3.23.0
|
|
- secretstorage==3.4.0
|
|
- urllib3==2.5.0
|
|
- websockets==15.0.1
|
|
state: present
|
|
extra_args: --break-system-packages
|
|
become: yes
|
|
|
|
- name: Upgrade yt-dlp and bgutil provider
|
|
ansible.builtin.shell: |
|
|
set -e
|
|
python3 -m pip install -U --pre "yt-dlp[default,curl-cffi]" --break-system-packages
|
|
python3 -m pip install --no-cache-dir -U bgutil-ytdlp-pot-provider --break-system-packages
|
|
args:
|
|
warn: false
|
|
become: yes
|
|
changed_when: true
|
|
|
|
- name: Check for FFmpeg
|
|
stat:
|
|
path: /usr/local/bin/ffmpeg
|
|
register: ffmpeg_binary
|
|
become: yes
|
|
|
|
- name: Install FFmpeg
|
|
when: not ffmpeg_binary.stat.exists
|
|
become: yes
|
|
block:
|
|
- name: Create ffmpeg directory
|
|
ansible.builtin.file:
|
|
path: /opt/ffmpeg
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Download and unarchive FFmpeg
|
|
ansible.builtin.unarchive:
|
|
src: "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz"
|
|
dest: /opt/ffmpeg
|
|
remote_src: yes
|
|
extra_opts: [--strip-components=1]
|
|
|
|
- name: Symlink ffmpeg and ffprobe
|
|
ansible.builtin.file:
|
|
src: "/opt/ffmpeg/bin/{{ item }}"
|
|
dest: "/usr/local/bin/{{ item }}"
|
|
state: link
|
|
force: yes
|
|
loop:
|
|
- ffmpeg
|
|
- ffprobe
|
|
|
|
- name: Check for Deno
|
|
stat:
|
|
path: /usr/local/bin/deno
|
|
register: deno_binary
|
|
become: yes
|
|
|
|
- name: Install Deno
|
|
when: not deno_binary.stat.exists
|
|
become: yes
|
|
block:
|
|
- name: Download and unarchive Deno
|
|
ansible.builtin.unarchive:
|
|
src: https://github.com/denoland/deno/releases/latest/download/deno-x86_64-unknown-linux-gnu.zip
|
|
dest: /usr/local/bin/
|
|
remote_src: yes
|
|
mode: '0755'
|
|
|
|
- name: Check if ytops_client requirements.txt exists
|
|
stat:
|
|
path: "{{ airflow_worker_dir }}/ytops_client/requirements.txt"
|
|
register: ytops_client_reqs
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
|
|
- name: Install dependencies from ytops_client/requirements.txt
|
|
ansible.builtin.pip:
|
|
requirements: "{{ airflow_worker_dir }}/ytops_client/requirements.txt"
|
|
state: present
|
|
extra_args: --break-system-packages
|
|
when: ytops_client_reqs.stat.exists
|
|
become: yes
|
|
|
|
# Include Docker health check
|
|
- name: Include Docker health check tasks
|
|
include_tasks: tasks/docker_health_check.yml
|
|
|
|
|
|
- name: Pull pre-built Docker images for ytdlp-ops services
|
|
ansible.builtin.command: >
|
|
docker compose --project-directory . -f configs/docker-compose-ytdlp-ops.yaml pull --ignore-buildable
|
|
args:
|
|
chdir: "{{ airflow_worker_dir }}"
|
|
become: yes
|
|
become_user: "{{ ansible_user }}"
|
|
register: docker_pull_result
|
|
retries: 3
|
|
delay: 10
|
|
changed_when: "'Pulling' in docker_pull_result.stdout or 'Downloaded' in docker_pull_result.stdout"
|
|
|
|
- name: Show docker pull output
|
|
ansible.builtin.debug:
|
|
var: docker_pull_result.stdout_lines
|
|
when: docker_pull_result.changed
|
|
|
|
roles:
|
|
- ytdlp-worker
|