164 lines
4.4 KiB
YAML
164 lines
4.4 KiB
YAML
---
|
|
- name: Check if Airflow worker deployment directory exists
|
|
stat:
|
|
path: "{{ airflow_worker_dir }}"
|
|
register: worker_dir_stat
|
|
|
|
- name: Ensure Airflow worker deployment directory exists
|
|
file:
|
|
path: "{{ airflow_worker_dir }}"
|
|
state: directory
|
|
owner: "{{ ssh_user }}"
|
|
group: ytdl
|
|
mode: '0755'
|
|
become: yes
|
|
when: not worker_dir_stat.stat.exists
|
|
|
|
- name: "Log: Syncing Airflow core files"
|
|
debug:
|
|
msg: "Syncing DAGs, configs, and Python source code to the worker node."
|
|
|
|
- name: Sync Airflow worker files
|
|
synchronize:
|
|
src: "../{{ item }}"
|
|
dest: "{{ airflow_worker_dir }}/"
|
|
archive: yes
|
|
recursive: yes
|
|
rsync_path: "sudo rsync"
|
|
rsync_opts: "{{ rsync_default_opts }}"
|
|
loop:
|
|
- "airflow/Dockerfile"
|
|
- "airflow/.dockerignore"
|
|
- "airflow/dags"
|
|
- "airflow/config"
|
|
- "airflow/airflow.cfg"
|
|
- "yt_ops_package/setup.py"
|
|
- "yt_ops_package/yt_ops_services"
|
|
- "yt_ops_package/thrift_model"
|
|
- "yt_ops_package/VERSION"
|
|
- "yt_ops_package/pangramia"
|
|
- "airflow/init-airflow.sh"
|
|
- "airflow/update-yt-dlp.sh"
|
|
- "yt_ops_package/get_info_json_client.py"
|
|
- "yt_ops_package/proxy_manager_client.py"
|
|
- "token_generator"
|
|
- "utils"
|
|
|
|
- name: Check if source directories exist
|
|
stat:
|
|
path: "../{{ item }}"
|
|
register: source_dirs
|
|
loop:
|
|
- "airflow/inputfiles"
|
|
- "airflow/plugins"
|
|
- "airflow/addfiles"
|
|
- "airflow/bgutil-ytdlp-pot-provider"
|
|
|
|
- name: Sync optional directories if they exist
|
|
synchronize:
|
|
src: "../{{ item.item }}/"
|
|
dest: "{{ airflow_worker_dir }}/{{ item.item | basename }}/"
|
|
archive: yes
|
|
recursive: yes
|
|
delete: yes
|
|
rsync_path: "sudo rsync"
|
|
rsync_opts: "{{ rsync_default_opts }}"
|
|
loop: "{{ source_dirs.results }}"
|
|
when: item.stat.exists
|
|
|
|
- name: Sync pangramia thrift files
|
|
synchronize:
|
|
src: "../yt_ops_package/thrift_model/gen_py/pangramia/"
|
|
dest: "{{ airflow_worker_dir }}/pangramia/"
|
|
archive: yes
|
|
recursive: yes
|
|
delete: yes
|
|
rsync_path: "sudo rsync"
|
|
rsync_opts: "{{ rsync_default_opts }}"
|
|
|
|
- name: Template docker-compose file for worker
|
|
template:
|
|
src: "{{ playbook_dir }}/../airflow/docker-compose-dl.yaml.j2"
|
|
dest: "{{ airflow_worker_dir }}/docker-compose-dl.yaml"
|
|
mode: "{{ file_permissions }}"
|
|
owner: "{{ ssh_user }}"
|
|
group: ytdl
|
|
become: yes
|
|
|
|
- name: Create .env file for Airflow worker service
|
|
template:
|
|
src: "../../templates/.env.worker.j2"
|
|
dest: "{{ airflow_worker_dir }}/.env"
|
|
mode: "{{ file_permissions }}"
|
|
owner: "{{ ssh_user }}"
|
|
group: ytdl
|
|
become: yes
|
|
vars:
|
|
service_role: "worker"
|
|
|
|
- name: Create symlink for docker-compose.yaml
|
|
file:
|
|
src: "{{ airflow_worker_dir }}/docker-compose-dl.yaml"
|
|
dest: "{{ airflow_worker_dir }}/docker-compose.yaml"
|
|
state: link
|
|
owner: "{{ ssh_user }}"
|
|
group: ytdl
|
|
follow: no
|
|
|
|
- name: Ensure correct permissions for build context
|
|
file:
|
|
path: "{{ airflow_worker_dir }}"
|
|
state: directory
|
|
owner: "{{ ssh_user }}"
|
|
group: ytdl
|
|
recurse: yes
|
|
become: yes
|
|
|
|
- name: Verify Dockerfile exists in build directory
|
|
stat:
|
|
path: "{{ airflow_worker_dir }}/Dockerfile"
|
|
register: dockerfile_stat
|
|
|
|
- name: Fail if Dockerfile is missing
|
|
fail:
|
|
msg: "Dockerfile not found in {{ airflow_worker_dir }}. Cannot build image."
|
|
when: not dockerfile_stat.stat.exists
|
|
|
|
- name: "Log: Building Airflow Docker image"
|
|
debug:
|
|
msg: "Building the main Airflow Docker image ({{ airflow_image_name }}) locally on the worker node. This may take a few minutes."
|
|
|
|
- name: Build Airflow worker image
|
|
community.docker.docker_image:
|
|
name: "{{ airflow_image_name }}"
|
|
build:
|
|
path: "{{ airflow_worker_dir }}"
|
|
dockerfile: "Dockerfile"
|
|
source: build
|
|
force_source: true
|
|
|
|
- name: Make Airflow init script executable
|
|
file:
|
|
path: "{{ airflow_worker_dir }}/init-airflow.sh"
|
|
mode: "0755"
|
|
become: yes
|
|
|
|
- name: Run Airflow init script
|
|
shell:
|
|
cmd: "./init-airflow.sh"
|
|
chdir: "{{ airflow_worker_dir }}"
|
|
become: yes
|
|
become_user: "{{ ssh_user }}"
|
|
|
|
- name: "Log: Starting Airflow services"
|
|
debug:
|
|
msg: "Starting Airflow worker services (celery worker) on the node using docker-compose."
|
|
|
|
- name: Start Airflow worker service
|
|
community.docker.docker_compose_v2:
|
|
project_src: "{{ airflow_worker_dir }}"
|
|
files:
|
|
- "docker-compose-dl.yaml"
|
|
state: present
|
|
remove_orphans: true
|