65 lines
2.6 KiB
YAML
65 lines
2.6 KiB
YAML
# Policy: Queue-based Download Simulation
|
|
#
|
|
# This policy simulates a continuous stream of download requests
|
|
# by pulling tasks from a Redis queue, downloading the specified formats,
|
|
# and pushing results to appropriate result queues.
|
|
#
|
|
name: queue_download_simulation
|
|
|
|
settings:
|
|
mode: download_only
|
|
orchestration_mode: queue_download
|
|
profile_mode: from_pool_with_lock
|
|
|
|
dummy_simulation_settings:
|
|
download_min_seconds: 5
|
|
download_max_seconds: 8
|
|
download_failure_rate: 0.0
|
|
download_skipped_failure_rate: 0.0
|
|
|
|
|
|
execution_control:
|
|
workers: 4
|
|
# How long a worker should pause if it cannot find an available profile or task.
|
|
worker_polling_interval_seconds: 1
|
|
# Run until conditions
|
|
run_until:
|
|
# Run for this many minutes (0 = unlimited)
|
|
minutes: 0
|
|
# Process this many requests (0 = unlimited)
|
|
requests: 0
|
|
|
|
download_policy:
|
|
profile_prefix: "user1"
|
|
# Default cooldown in seconds if not specified by the enforcer in Redis.
|
|
# The value from Redis (set via `unlock_cooldown_seconds` in the enforcer policy)
|
|
# will always take precedence. This is a fallback.
|
|
# Can be an integer (e.g., 1) or a range (e.g., [1, 3]).
|
|
default_unlock_cooldown_seconds: 1
|
|
# Directory to save downloaded files
|
|
output_dir: "downloaded_media/queue_downloads"
|
|
# Extra arguments to pass to the download command
|
|
extra_args: "--verbose"
|
|
|
|
# After a download task is successfully processed, rename the source info.json
|
|
# to prevent re-processing. This is safe if you generate one download task per info.json.
|
|
rename_source_info_json_on_success: true
|
|
|
|
# --- Airflow Integration ---
|
|
# If true, move downloaded media and info.json to a timestamped, video-id-based
|
|
# directory structure that the Airflow DAGs can process.
|
|
output_to_airflow_ready_dir: true
|
|
airflow_ready_dir_base_path: "downloadfiles/videos/ready"
|
|
|
|
queue_policy:
|
|
# Set to false to use legacy, unprefixed queue names (e.g., 'queue2_dl_inbox').
|
|
# Set to true (or omit) to use environment-prefixed names (e.g., 'sim_download_queue2_dl_inbox').
|
|
use_env_prefix: false
|
|
|
|
# How many tasks to process in a batch. For downloads, this should be 1,
|
|
# as each worker locks a profile for a single download task.
|
|
batch_size: 1
|
|
|
|
simulation_parameters:
|
|
download_env: "sim_download"
|