# Policy: Queue-based Download Simulation # # This policy simulates a continuous stream of download requests # by pulling tasks from a Redis queue, downloading the specified formats, # and pushing results to appropriate result queues. # name: queue_download_simulation settings: mode: download_only orchestration_mode: queue_download profile_mode: from_pool_with_lock dummy_simulation_settings: download_min_seconds: 5 download_max_seconds: 8 download_failure_rate: 0.0 download_skipped_failure_rate: 0.0 execution_control: workers: 4 # How long a worker should pause if it cannot find an available profile or task. worker_polling_interval_seconds: 1 # Run until conditions run_until: # Run for this many minutes (0 = unlimited) minutes: 0 # Process this many requests (0 = unlimited) requests: 0 download_policy: profile_prefix: "user1" # Default cooldown in seconds if not specified by the enforcer in Redis. # The value from Redis (set via `unlock_cooldown_seconds` in the enforcer policy) # will always take precedence. This is a fallback. # Can be an integer (e.g., 1) or a range (e.g., [1, 3]). default_unlock_cooldown_seconds: 1 # Directory to save downloaded files output_dir: "downloaded_media/queue_downloads" # Extra arguments to pass to the download command extra_args: "--verbose" # After a download task is successfully processed, rename the source info.json # to prevent re-processing. This is safe if you generate one download task per info.json. rename_source_info_json_on_success: true # --- Airflow Integration --- # If true, move downloaded media and info.json to a timestamped, video-id-based # directory structure that the Airflow DAGs can process. output_to_airflow_ready_dir: true airflow_ready_dir_base_path: "downloadfiles/videos/ready" queue_policy: # Set to false to use legacy, unprefixed queue names (e.g., 'queue2_dl_inbox'). # Set to true (or omit) to use environment-prefixed names (e.g., 'sim_download_queue2_dl_inbox'). use_env_prefix: false # How many tasks to process in a batch. For downloads, this should be 1, # as each worker locks a profile for a single download task. batch_size: 1 simulation_parameters: download_env: "sim_download"