yt-dlp-dags/ytops_client-source/policies/8_unified_simulation_enforcer.yaml

# Policy for the unified simulation enforcer.
# This file is used by `bin/ytops-client policy-enforcer --live` to manage
# both the authentication and download simulation environments from a single process.

simulation_parameters:
  # --- Common Redis settings for all tools ---
  # The enforcer will connect to two different Redis environments (key prefixes)
  # based on these settings, applying the corresponding policies to each.
  env_file: ".env"
  auth_env: "sim_auth"
  download_env: "sim_download"
  
  # How often the enforcer should wake up and apply all policies.
  interval_seconds: 2

# --- Policies for the Authentication Simulation ---
auth_policy_enforcer_config:

  # Ban if 2 failures occur within a 1-minute window.
  #ban_on_failures: 2
  #ban_on_failures_window_minutes: 1
  
  # The standard rest policy is disabled, as rotation is handled by the profile group.
  
  # New rate limit policy to enforce requests-per-hour limits.
  # For guest sessions, the limit is ~300 videos/hour.
  rate_limit_requests: 0
  rate_limit_window_minutes: 60
  rate_limit_rest_duration_minutes: 5
                        
  rest_after_requests: 0
  rest_duration_minutes: 10

  # NOTE on Rate Limits: With the default yt-dlp settings, the rate limit for guest
  # sessions is ~300 videos/hour (~1000 webpage/player requests per hour).
  # For accounts, it is ~2000 videos/hour (~4000 webpage/player requests per hour).
  # The settings below should be configured to respect these limits.
  
  # New setting for load balancing across profile groups.
  # "longest_idle": Activates the profile that has been idle the longest across all groups (based on last_used time).
  #                 This is a global FIFO strategy that effectively cycles through profiles regardless of their group.
  # "least_loaded": Prioritizes activating a profile from the group with the fewest pending downloads.
  #                 If multiple groups have zero pending downloads, it acts as a FIFO queue, activating
  #                 the one that finished its last download batch the earliest. This is useful when you want
  #                 to ensure a group finishes its entire workload before another group starts.
  profile_selection_strategy: "longest_idle"
  
  # Enforce a total limit of active profiles across all groups defined below.
  # Set to 1 to ensure only one group's profile is active at any time.
  global_max_active_profiles: 1

  # Define separate profile groups for each user type.
  # This allows one profile from each group to be active simultaneously,
  # ensuring the single auth worker is never blocked waiting for downloads.
  profile_groups:
    - name: "auth_user1"
      prefix: "user1"
      max_active_profiles: 1
      rotate_after_requests: 5
      rest_duration_minutes_on_rotation: 0.20
      wait_download_finish_per_group: true
      max_wait_for_downloads_minutes: 240
    - name: "auth_user2"
      prefix: "user2"
      max_active_profiles: 1
      rotate_after_requests: 5
      rest_duration_minutes_on_rotation: 0.20
      wait_download_finish_per_group: true
      max_wait_for_downloads_minutes: 240
    - name: "auth_user3"
      prefix: "user3"
      max_active_profiles: 1
      rotate_after_requests: 5
      rest_duration_minutes_on_rotation: 0.20
      wait_download_finish_per_group: true
      max_wait_for_downloads_minutes: 240

  proxy_work_minutes: 0
  proxy_rest_duration_minutes: 0

  # Global maximum time a proxy can be active before being rested, regardless of
  # other rules. Acts as a safety net. Set to 0 to disable.
  max_global_proxy_active_minutes: 0
  rest_duration_on_max_active: 10

  # Proxy-level ban on failure burst is disabled.
  proxy_ban_on_failures: 0
  proxy_ban_window_minutes: 2

  # Clean up locks held for more than 16 minutes (960s) to prevent stuck workers.
  # This should be longer than the docker container timeout (15m).
  unlock_stale_locks_after_seconds: 960

  # A short post-task cooldown for auth simulation profiles. When a batch is finished,
  # the profile is put into COOLDOWN briefly. This prevents a worker from immediately
  # re-locking the same profile, giving the policy enforcer a window to perform rotation.
  unlock_cooldown_seconds: 0

# --- Cross-simulation synchronization ---
cross_simulation_sync:
  # Link auth profiles to download profiles (by prefix)
  profile_links:
    - auth: "user1"
      download: "user1"
    - auth: "user2"
      download: "user2"
    - auth: "user3"
      download: "user3"

  # Which states to synchronize from auth to download.
  # 'RESTING' is no longer needed here; the new group-aware deactivation logic
  # in `sync_active_profile` handles rotation more cleanly.
  sync_states:
    - "BANNED"

  # If true, when an auth profile is rotated, the corresponding
  # download profile group will also be rotated. This is now handled by the
  # group-aware deactivation logic triggered by `sync_active_profile`.
  sync_rotation: true

  # If true, a BANNED state on an auth profile will force the download profile
  # to also be BANNED.
  enforce_auth_lead: true

  # CRITICAL: Ensures the correct download profile GROUP is active.
  # This will activate the target download profile and rest any profiles in other groups.
  sync_active_profile: true

  # When an auth profile is in the 'waiting_downloads' state, ensure the
  # matching download profile is active so it can process those downloads.
  sync_waiting_downloads: true

# --- Policies for the Download Simulation ---
download_policy_enforcer_config:
  # Ban if 1 failure occurs within a 1-minute window.
  ban_on_failures: 1
  ban_on_failures_window_minutes: 1

  # Standard rest policy is disabled in favor of group rotation.
  
  # New rate limit policy to enforce requests-per-hour limits.
  # For guest sessions, the limit is ~300 videos/hour. We set it slightly lower to be safe.
  rate_limit_requests: 280
  rate_limit_window_minutes: 60
  rate_limit_rest_duration_minutes: 5
  rest_after_requests: 0
  rest_duration_minutes: 20

  # NOTE on Rate Limits: With the default yt-dlp settings, the rate limit for guest
  # sessions is ~300 videos/hour (~1000 webpage/player requests per hour).
  # For accounts, it is ~2000 videos/hour (~4000 webpage/player requests per hour).
  # The settings below should be configured to respect these limits.

  # Define separate profile groups for download workers.
  # Increase max_active_profiles to allow all profiles in a group to be used.
  profile_groups:
    - name: "download_user1"
      prefix: "user1"
      rotate_after_requests: 0
      rest_duration_minutes_on_rotation: 0.2
      # max_active_profiles: 0 # Allow all profiles in this group to be active (0, -1, or omitted)
    - name: "download_user2"
      prefix: "user2"
      rotate_after_requests: 0
      rest_duration_minutes_on_rotation: 0.2
      # max_active_profiles: 0 # Allow all profiles in this group to be active (0, -1, or omitted)
    - name: "download_user3"
      prefix: "user3"
      rotate_after_requests: 0
      rest_duration_minutes_on_rotation: 0.2
      # max_active_profiles: 0 # Allow all profiles in this group to be active (0, -1, or omitted)

  # Time-based proxy rules are disabled.
  proxy_work_minutes: 0
  proxy_rest_duration_minutes: 10
  
  # Global maximum time a proxy can be active before being rested, regardless of
  # other rules. Acts as a safety net. Set to 0 to disable.
  max_global_proxy_active_minutes: 0
  rest_duration_on_max_active: 10

  # Proxy-level ban on failure burst is disabled.
  proxy_ban_on_failures: 3
  proxy_ban_window_minutes: 1

  # Clean up download locks held for more than 16 minutes (960s) to allow for long downloads.
  # This should be longer than the docker container timeout (15m).
  unlock_stale_locks_after_seconds: 960

  # After a profile is used for a download, unlock it but put it in COOLDOWN
  # state for 2-3s. This is enforced by the worker, which reads this config from Redis.
  unlock_cooldown_seconds: [2, 3]