373 lines
20 KiB
YAML
373 lines
20 KiB
YAML
---
|
|
- name: "STRESS-SETUP: Manage full worker lifecycle based on inventory"
|
|
hosts: workers
|
|
gather_facts: no
|
|
vars:
|
|
# Default action
|
|
action: "status" # Available actions: start, stop, status, start-auth, stop-auth, start-download, stop-download, stop-generator
|
|
|
|
tasks:
|
|
- name: "Start all configured generators and simulators"
|
|
when: action == "start"
|
|
block:
|
|
- name: "Set combined profile prefixes string"
|
|
ansible.builtin.set_fact:
|
|
combined_prefixes: "{{ profile_prefixes | default([]) | join(',') }}"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
|
|
- name: "Start auth generator(s)"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
block:
|
|
- name: "Start single auth generator for all profiles: {{ combined_prefixes | default('none') }}"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-auth-generator.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_generator=true"
|
|
-e "profile_prefix={{ combined_prefixes }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if auth_min_seconds is defined %}-e "auth_min_seconds={{ auth_min_seconds }}"{% endif %}
|
|
{% if auth_max_seconds is defined %}-e "auth_max_seconds={{ auth_max_seconds }}"{% endif %}
|
|
{% if batch_size is defined %}-e "batch_size={{ batch_size }}"{% endif %}
|
|
{% if create_download_tasks is defined %}-e "create_download_tasks={{ create_download_tasks }}"{% endif %}
|
|
{% if formats_to_download is defined %}-e "formats_to_download={{ formats_to_download }}"{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
when: (auth_workers_per_profile | default(0) | int == 0) and (auth_workers_total | default(0) | int > 0)
|
|
|
|
- name: "Start parallel auth generators for each profile"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-auth-generator.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_generator=true"
|
|
-e "profile_prefix={{ item }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if auth_min_seconds is defined %}-e "auth_min_seconds={{ auth_min_seconds }}"{% endif %}
|
|
{% if auth_max_seconds is defined %}-e "auth_max_seconds={{ auth_max_seconds }}"{% endif %}
|
|
{% if batch_size is defined %}-e "batch_size={{ batch_size }}"{% endif %}
|
|
{% if create_download_tasks is defined %}-e "create_download_tasks={{ create_download_tasks }}"{% endif %}
|
|
{% if formats_to_download is defined %}-e "formats_to_download={{ formats_to_download }}"{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
loop: "{{ profile_prefixes }}"
|
|
loop_control:
|
|
loop_var: item
|
|
label: "profile: {{ item }}"
|
|
when: auth_workers_per_profile | default(0) | int > 0
|
|
|
|
- name: "Start download simulator(s)"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
block:
|
|
- name: "Start single download simulator for all profiles: {{ combined_prefixes | default('none') }}"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-download-simulation.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_download=true"
|
|
-e "profile_prefix={{ combined_prefixes }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if download_min_seconds is defined %}-e "download_min_seconds={{ download_min_seconds }}"{% endif %}
|
|
{% if download_max_seconds is defined %}-e "download_max_seconds={{ download_max_seconds }}"{% endif %}
|
|
{% if extra_set_args is defined %}-e 'extra_set_args={{ extra_set_args | to_json }}'{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
when: (download_workers_per_profile | default(0) | int == 0) and (download_workers_total | default(0) | int > 0)
|
|
|
|
- name: "Start parallel download simulators for each profile"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-download-simulation.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_download=true"
|
|
-e "profile_prefix={{ item }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if download_min_seconds is defined %}-e "download_min_seconds={{ download_min_seconds }}"{% endif %}
|
|
{% if download_max_seconds is defined %}-e "download_max_seconds={{ download_max_seconds }}"{% endif %}
|
|
{% if extra_set_args is defined %}-e 'extra_set_args={{ extra_set_args | to_json }}'{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
loop: "{{ profile_prefixes }}"
|
|
loop_control:
|
|
loop_var: item
|
|
label: "profile: {{ item }}"
|
|
when: download_workers_per_profile | default(0) | int > 0
|
|
|
|
- name: "Start only auth generators on workers"
|
|
when: action == "start-auth"
|
|
block:
|
|
- name: "Set combined profile prefixes string"
|
|
ansible.builtin.set_fact:
|
|
combined_prefixes: "{{ profile_prefixes | default([]) | join(',') }}"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
|
|
- name: "Start auth generator(s)"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
block:
|
|
- name: "Start single auth generator for all profiles: {{ combined_prefixes | default('none') }}"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-auth-generator.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_generator=true"
|
|
-e "profile_prefix={{ combined_prefixes }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if auth_min_seconds is defined %}-e "auth_min_seconds={{ auth_min_seconds }}"{% endif %}
|
|
{% if auth_max_seconds is defined %}-e "auth_max_seconds={{ auth_max_seconds }}"{% endif %}
|
|
{% if batch_size is defined %}-e "batch_size={{ batch_size }}"{% endif %}
|
|
{% if create_download_tasks is defined %}-e "create_download_tasks={{ create_download_tasks }}"{% endif %}
|
|
{% if formats_to_download is defined %}-e "formats_to_download={{ formats_to_download }}"{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
when: (auth_workers_per_profile | default(0) | int == 0) and (auth_workers_total | default(0) | int > 0)
|
|
|
|
- name: "Start parallel auth generators for each profile"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-auth-generator.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_generator=true"
|
|
-e "profile_prefix={{ item }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if auth_min_seconds is defined %}-e "auth_min_seconds={{ auth_min_seconds }}"{% endif %}
|
|
{% if auth_max_seconds is defined %}-e "auth_max_seconds={{ auth_max_seconds }}"{% endif %}
|
|
{% if batch_size is defined %}-e "batch_size={{ batch_size }}"{% endif %}
|
|
{% if create_download_tasks is defined %}-e "create_download_tasks={{ create_download_tasks }}"{% endif %}
|
|
{% if formats_to_download is defined %}-e "formats_to_download={{ formats_to_download }}"{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
loop: "{{ profile_prefixes }}"
|
|
loop_control:
|
|
loop_var: item
|
|
label: "profile: {{ item }}"
|
|
when: auth_workers_per_profile | default(0) | int > 0
|
|
|
|
- name: "Start only download simulators on workers"
|
|
when: action == "start-download"
|
|
block:
|
|
- name: "Set combined profile prefixes string"
|
|
ansible.builtin.set_fact:
|
|
combined_prefixes: "{{ profile_prefixes | default([]) | join(',') }}"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
|
|
- name: "Start download simulator(s)"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
block:
|
|
- name: "Start single download simulator for all profiles: {{ combined_prefixes | default('none') }}"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-download-simulation.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_download=true"
|
|
-e "profile_prefix={{ combined_prefixes }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if download_min_seconds is defined %}-e "download_min_seconds={{ download_min_seconds }}"{% endif %}
|
|
{% if download_max_seconds is defined %}-e "download_max_seconds={{ download_max_seconds }}"{% endif %}
|
|
{% if extra_set_args is defined %}-e 'extra_set_args={{ extra_set_args | to_json }}'{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
when: (download_workers_per_profile | default(0) | int == 0) and (download_workers_total | default(0) | int > 0)
|
|
|
|
- name: "Start parallel download simulators for each profile"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-download-simulation.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "start_download=true"
|
|
-e "profile_prefix={{ item }}"
|
|
{% if dummy_batch is defined %}-e "dummy_batch={{ dummy_batch }}"{% endif %}
|
|
{% if download_min_seconds is defined %}-e "download_min_seconds={{ download_min_seconds }}"{% endif %}
|
|
{% if download_max_seconds is defined %}-e "download_max_seconds={{ download_max_seconds }}"{% endif %}
|
|
{% if extra_set_args is defined %}-e 'extra_set_args={{ extra_set_args | to_json }}'{% endif %}
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
loop: "{{ profile_prefixes }}"
|
|
loop_control:
|
|
loop_var: item
|
|
label: "profile: {{ item }}"
|
|
when: download_workers_per_profile | default(0) | int > 0
|
|
|
|
- name: "Stop only auth generators on workers (via playbook call)"
|
|
when: action == "stop-generator"
|
|
block:
|
|
- name: "Set combined profile prefixes string"
|
|
ansible.builtin.set_fact:
|
|
combined_prefixes: "{{ profile_prefixes | default([]) | join(',') }}"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
|
|
- name: "Stop single auth generator for profiles: {{ combined_prefixes | default('none') }}"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-auth-generator.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "stop_generator=true"
|
|
-e "profile_prefix={{ combined_prefixes }}"
|
|
delegate_to: localhost
|
|
changed_when: true
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
|
|
- name: "Stop only auth generators on workers"
|
|
when: action == "stop-auth"
|
|
block:
|
|
- name: Kill all auth generator tmux sessions on this worker
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
for session in $(tmux list-sessions -F "#{session_name}" 2>/dev/null | grep -E "^stress-auth-"); do
|
|
tmux kill-session -t "$session"
|
|
done || true
|
|
ignore_errors: yes
|
|
changed_when: false
|
|
|
|
- name: Kill all ytops-client auth generator processes on this worker
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
# Gracefully terminate
|
|
ps aux | grep "[y]tops-client.*stress-policy.*12_queue_auth_simulation" | awk '{print $2}' | xargs kill >/dev/null 2>&1 || true
|
|
sleep 0.5
|
|
# Force kill
|
|
ps aux | grep "[y]tops-client.*stress-policy.*12_queue_auth_simulation" | awk '{print $2}' | xargs kill -9 >/dev/null 2>&1 || true
|
|
ignore_errors: yes
|
|
changed_when: false
|
|
|
|
- name: "Stop only download simulators on workers"
|
|
when: action == "stop-download"
|
|
block:
|
|
- name: Kill all download simulator tmux sessions on this worker
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
for session in $(tmux list-sessions -F "#{session_name}" 2>/dev/null | grep -E "^stress-download-"); do
|
|
tmux kill-session -t "$session"
|
|
done || true
|
|
ignore_errors: yes
|
|
changed_when: false
|
|
|
|
- name: Kill all ytops-client download simulator processes on this worker
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
# Gracefully terminate
|
|
ps aux | grep "[y]tops-client.*stress-policy.*11_direct_docker_download_simulation" | awk '{print $2}' | xargs kill >/dev/null 2>&1 || true
|
|
sleep 0.5
|
|
# Force kill
|
|
ps aux | grep "[y]tops-client.*stress-policy.*11_direct_docker_download_simulation" | awk '{print $2}' | xargs kill -9 >/dev/null 2>&1 || true
|
|
ignore_errors: yes
|
|
changed_when: false
|
|
|
|
- name: "Stop all worker generators and simulators"
|
|
when: action == "stop"
|
|
block:
|
|
- name: Kill all tmux sessions starting with 'stress-' on this worker
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
for session in $(tmux list-sessions -F "#{session_name}" 2>/dev/null | grep -E "^stress-"); do
|
|
tmux kill-session -t "$session"
|
|
done || true
|
|
ignore_errors: yes
|
|
changed_when: false
|
|
|
|
- name: Kill all ytops-client processes on this worker
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
# Gracefully terminate
|
|
ps aux | grep "[y]tops-client.*stress-policy" | awk '{print $2}' | xargs kill >/dev/null 2>&1 || true
|
|
sleep 0.5
|
|
# Force kill
|
|
ps aux | grep "[y]tops-client.*stress-policy" | awk '{print $2}' | xargs kill -9 >/dev/null 2>&1 || true
|
|
ignore_errors: yes
|
|
changed_when: false
|
|
|
|
- name: "Check status of all configured generators and simulators"
|
|
when: action == "status"
|
|
block:
|
|
- name: "Set combined profile prefixes string"
|
|
ansible.builtin.set_fact:
|
|
combined_prefixes: "{{ profile_prefixes | default([]) | join(',') }}"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
|
|
- name: "Check auth generator status"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
block:
|
|
- name: "Check single auth generator for all profiles: {{ combined_prefixes | default('none') }}"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-auth-generator.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "check_status=true"
|
|
-e "profile_prefix={{ combined_prefixes }}"
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
register: auth_status_check_combined
|
|
when: (auth_workers_per_profile | default(0) | int == 0) and (auth_workers_total | default(0) | int > 0)
|
|
|
|
- name: "Display combined auth generator status for {{ inventory_hostname }}"
|
|
ansible.builtin.debug:
|
|
var: auth_status_check_combined.stdout_lines
|
|
when: auth_status_check_combined is defined and auth_status_check_combined.stdout_lines is defined
|
|
|
|
- name: "Check parallel auth generators for each profile"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-auth-generator.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "check_status=true"
|
|
-e "profile_prefix={{ item }}"
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
loop: "{{ profile_prefixes }}"
|
|
loop_control:
|
|
loop_var: item
|
|
label: "profile: {{ item }}"
|
|
register: auth_status_check_parallel
|
|
when: auth_workers_per_profile | default(0) | int > 0
|
|
|
|
- name: "Display parallel auth generator status for {{ inventory_hostname }}"
|
|
ansible.builtin.debug:
|
|
msg: "{{ item.stdout_lines }}"
|
|
loop: "{{ auth_status_check_parallel.results | default([]) }}"
|
|
loop_control:
|
|
label: "status for profile: {{ item.item }}"
|
|
when: auth_status_check_parallel is defined and auth_status_check_parallel.results is defined
|
|
|
|
- name: "Check download simulator status"
|
|
when: profile_prefixes is defined and profile_prefixes | length > 0
|
|
block:
|
|
- name: "Check single download simulator for all profiles: {{ combined_prefixes | default('none') }}"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-download-simulation.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "check_status=true"
|
|
-e "profile_prefix={{ combined_prefixes }}"
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
register: download_status_check_combined
|
|
when: (download_workers_per_profile | default(0) | int == 0) and (download_workers_total | default(0) | int > 0)
|
|
|
|
- name: "Display combined download simulator status for {{ inventory_hostname }}"
|
|
ansible.builtin.debug:
|
|
var: download_status_check_combined.stdout_lines
|
|
when: download_status_check_combined is defined and download_status_check_combined.stdout_lines is defined
|
|
|
|
- name: "Check parallel download simulators for each profile"
|
|
ansible.builtin.command: >-
|
|
ansible-playbook {{ playbook_dir }}/playbook-stress-download-simulation.yml
|
|
-i {{ inventory_file }}
|
|
--limit {{ inventory_hostname }}
|
|
-e "check_status=true"
|
|
-e "profile_prefix={{ item }}"
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
loop: "{{ profile_prefixes }}"
|
|
loop_control:
|
|
loop_var: item
|
|
label: "profile: {{ item }}"
|
|
register: download_status_check_parallel
|
|
when: download_workers_per_profile | default(0) | int > 0
|
|
|
|
- name: "Display parallel download simulator status for {{ inventory_hostname }}"
|
|
ansible.builtin.debug:
|
|
msg: "{{ item.stdout_lines }}"
|
|
loop: "{{ download_status_check_parallel.results | default([]) }}"
|
|
loop_control:
|
|
label: "status for profile: {{ item.item }}"
|
|
when: download_status_check_parallel is defined and download_status_check_parallel.results is defined
|