alex_p/yt-dlp-dags
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update chrome versions and remove XDG duplicates

Browse Source
This commit is contained in:
aperez 2025-12-26 14:09:39 +03:00
parent e284099eee
commit fa7c46f671
3 changed files with 40 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ytops_client-source/policies/10_direct_docker_auth_simulation.yaml
View File

@ -66,8 +66,10 @@ direct_docker_cli_policy:
# Template for generating User-Agent strings for new profiles. # Template for generating User-Agent strings for new profiles.
# The '{major_version}' will be replaced by a version string. # The '{major_version}' will be replaced by a version string.
user_agent_template: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major_version}.0.0.0 Safari/537.36" user_agent_template: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major_version}.0.0.0 Safari/537.36"
# Range of Chrome major versions to use for the template. A range suitable for TV devices. # Range of Chrome major versions to use for the template.
user_agent_version_range: [110, 120] # See CHROME_MAJOR_VERSION_RANGE in yt-dlp's random_user_agent():
# https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/utils/networking.py
user_agent_version_range: [137, 143]
batch_size: 25 batch_size: 25

6
ytops_client-source/policies/12_queue_auth_simulation.yaml
View File

@ -61,8 +61,10 @@ direct_docker_cli_policy:
# Template for generating User-Agent strings for new profiles. # Template for generating User-Agent strings for new profiles.
# The '{major_version}' will be replaced by a version string. # The '{major_version}' will be replaced by a version string.
user_agent_template: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major_version}.0.0.0 Safari/537.36" user_agent_template: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major_version}.0.0.0 Safari/537.36"
# Range of Chrome major versions to use for the template. A range suitable for TV devices. # Range of Chrome major versions to use for the template.
user_agent_version_range: [110, 120] # See CHROME_MAJOR_VERSION_RANGE in yt-dlp's random_user_agent():
# https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/utils/networking.py
user_agent_version_range: [137, 143]
# A base config file can be used, with overrides applied from the policy. # A base config file can be used, with overrides applied from the policy.
# The orchestrator will inject 'proxy', 'batch-file', and 'output' keys into the overrides. # The orchestrator will inject 'proxy', 'batch-file', and 'output' keys into the overrides.

53
ytops_client-source/ytops_client/stress_policy/workers.py
View File

@ -943,9 +943,13 @@ def _post_process_and_move_info_json(file_path, profile_name, proxy_url, policy,
added_count = state_manager.add_download_tasks_batch([download_task]) added_count = state_manager.add_download_tasks_batch([download_task])
if added_count > 0: if added_count > 0:
logger.info(f"[Worker {worker_id}] [{profile_name}] Added {added_count} download task(s) to queue for {video_id}") logger.info(f"[Worker {worker_id}] [{profile_name}] Added {added_count} download task(s) to queue for {video_id}")
else:
logger.error(f"[Worker {worker_id}] [{profile_name}] Failed to add download task to queue for {video_id}")
return False
except Exception as e: except Exception as e:
logger.error(f"[Worker {worker_id}] [{profile_name}] Failed to create download task for {video_id}: {e}", exc_info=True) logger.error(f"[Worker {worker_id}] [{profile_name}] Failed to create download task for {video_id}: {e}", exc_info=True)
return False
return True return True
except (IOError, json.JSONDecodeError, OSError) as e: except (IOError, json.JSONDecodeError, OSError) as e:
@ -1033,6 +1037,7 @@ def run_direct_batch_worker(worker_id, policy, state_manager, args, profile_mana
exec_control = policy.get('execution_control', {}) exec_control = policy.get('execution_control', {})
gen_policy = policy.get('info_json_generation_policy', {}) gen_policy = policy.get('info_json_generation_policy', {})
direct_policy = policy.get('direct_batch_cli_policy', {}) direct_policy = policy.get('direct_batch_cli_policy', {})
queue_policy = policy.get('queue_policy')
profile_prefix = gen_policy.get('profile_prefix') profile_prefix = gen_policy.get('profile_prefix')
if not profile_prefix: if not profile_prefix:
@ -1110,9 +1115,10 @@ def run_direct_batch_worker(worker_id, policy, state_manager, args, profile_mana
url_batch_len = len(url_batch) url_batch_len = len(url_batch)
batch_started = True batch_started = True
# Preemptively increment the counter to avoid race conditions with download workers. # Preemptively increment the counter if we expect to create download tasks.
profile_manager_instance.increment_pending_downloads(profile_name, url_batch_len) if queue_policy and queue_policy.get('formats_to_download'):
logger.info(f"[Worker {worker_id}] [{profile_name}] Preemptively incremented pending downloads by {url_batch_len} for the upcoming batch.") profile_manager_instance.increment_pending_downloads(profile_name, url_batch_len)
logger.info(f"[Worker {worker_id}] [{profile_name}] Preemptively incremented pending downloads by {url_batch_len} for the upcoming batch.")
end_idx = start_idx + len(url_batch) end_idx = start_idx + len(url_batch)
logger.info(f"[Worker {worker_id}] [{profile_name}] Processing batch of {len(url_batch)} URLs (lines {start_idx + 1}-{end_idx} from source).") logger.info(f"[Worker {worker_id}] [{profile_name}] Processing batch of {len(url_batch)} URLs (lines {start_idx + 1}-{end_idx} from source).")
@ -1564,8 +1570,8 @@ def run_direct_docker_worker(worker_id, policy, state_manager, args, profile_man
task_dir_name = os.path.basename(temp_task_dir_host) task_dir_name = os.path.basename(temp_task_dir_host)
task_dir_container = os.path.join(container_mount_path, task_dir_name) task_dir_container = os.path.join(container_mount_path, task_dir_name)
# Set XDG_CONFIG_HOME for yt-dlp to find the config automatically # The config file path is passed explicitly to yt-dlp via --config-locations,
environment['XDG_CONFIG_HOME'] = task_dir_container # so setting XDG_CONFIG_HOME in the environment is redundant.
# Write batch file # Write batch file
temp_batch_file_host = os.path.join(temp_task_dir_host, 'batch.txt') temp_batch_file_host = os.path.join(temp_task_dir_host, 'batch.txt')
@ -1701,8 +1707,7 @@ def run_direct_docker_worker(worker_id, policy, state_manager, args, profile_man
'mode': 'rw' 'mode': 'rw'
} }
# The command tells yt-dlp where to find the config file we created. # The command tells yt-dlp exactly where to find the config file we created.
# We still set XDG_CONFIG_HOME for any other config it might look for.
command = ['yt-dlp', '--config-locations', os.path.join(task_dir_container, 'yt-dlp/config')] command = ['yt-dlp', '--config-locations', os.path.join(task_dir_container, 'yt-dlp/config')]
logger.info(f"[Worker {worker_id}] [{profile_name}] Running docker command: {' '.join(shlex.quote(s) for s in command)}") logger.info(f"[Worker {worker_id}] [{profile_name}] Running docker command: {' '.join(shlex.quote(s) for s in command)}")
@ -1749,11 +1754,7 @@ def run_direct_docker_worker(worker_id, policy, state_manager, args, profile_man
# Success is the highest priority check # Success is the highest priority check
if '[info] Writing video metadata as JSON to:' in line: if '[info] Writing video metadata as JSON to:' in line:
with activity_lock: post_processed_successfully = False
live_success_count += 1
logger.info(f"[Worker {worker_id}] [{profile_name}] Live success #{live_success_count} detected from log.")
profile_manager_instance.record_activity(profile_name, 'success')
# --- Immediate post-processing --- # --- Immediate post-processing ---
try: try:
path_match = re.search(r"Writing video metadata as JSON to: '?([^']+)'?$", line) path_match = re.search(r"Writing video metadata as JSON to: '?([^']+)'?$", line)
@ -1774,7 +1775,7 @@ def run_direct_docker_worker(worker_id, policy, state_manager, args, profile_man
time.sleep(0.1) time.sleep(0.1)
if os.path.exists(host_file_path): if os.path.exists(host_file_path):
_post_process_and_move_info_json( post_processed_successfully = _post_process_and_move_info_json(
Path(host_file_path), profile_name, proxy_url, policy, worker_id, Path(host_file_path), profile_name, proxy_url, policy, worker_id,
state_manager, profile_manager_instance=profile_manager_instance state_manager, profile_manager_instance=profile_manager_instance
) )
@ -1782,6 +1783,16 @@ def run_direct_docker_worker(worker_id, policy, state_manager, args, profile_man
logger.warning(f"File from log not found on host for immediate processing: {host_file_path}") logger.warning(f"File from log not found on host for immediate processing: {host_file_path}")
except Exception as e: except Exception as e:
logger.error(f"Error during immediate post-processing from log line: {e}") logger.error(f"Error during immediate post-processing from log line: {e}")
with activity_lock:
if post_processed_successfully:
live_success_count += 1
logger.info(f"[Worker {worker_id}] [{profile_name}] Live success #{live_success_count} detected and post-processed.")
profile_manager_instance.record_activity(profile_name, 'success')
else:
live_failure_count += 1
logger.error(f"[Worker {worker_id}] [{profile_name}] Post-processing failed for a successful fetch. Recording as failure.")
profile_manager_instance.record_activity(profile_name, 'failure')
# --- End immediate post-processing --- # --- End immediate post-processing ---
return False return False
@ -1947,11 +1958,12 @@ def run_direct_docker_worker(worker_id, policy, state_manager, args, profile_man
finally: finally:
if locked_profile and batch_started: if locked_profile and batch_started:
# --- Reconcile pending downloads counter --- # --- Reconcile pending downloads counter ---
# This is in the finally block to guarantee it runs even if post-processing fails. if queue_policy and queue_policy.get('formats_to_download'):
adjustment = live_success_count - url_batch_len # This is in the finally block to guarantee it runs even if post-processing fails.
if adjustment != 0: adjustment = live_success_count - url_batch_len
logger.warning(f"[Worker {worker_id}] [{profile_name}] Reconciling pending downloads. Batch created {live_success_count}/{url_batch_len} files. Adjusting by {adjustment}.") if adjustment != 0:
profile_manager_instance.increment_pending_downloads(locked_profile['name'], adjustment) logger.warning(f"[Worker {worker_id}] [{profile_name}] Reconciling pending downloads. Batch created {live_success_count}/{url_batch_len} files. Adjusting by {adjustment}.")
profile_manager_instance.increment_pending_downloads(locked_profile['name'], adjustment)
if locked_profile: if locked_profile:
last_used_profile_name = locked_profile['name'] last_used_profile_name = locked_profile['name']
@ -2199,7 +2211,7 @@ def run_direct_docker_download_worker(worker_id, policy, state_manager, args, pr
config_dir_name = os.path.basename(temp_config_dir_host) config_dir_name = os.path.basename(temp_config_dir_host)
config_dir_container = os.path.join(container_mount_path, config_dir_name) config_dir_container = os.path.join(container_mount_path, config_dir_name)
environment = {'XDG_CONFIG_HOME': config_dir_container} environment = {}
base_config_content = "" base_config_content = ""
base_config_file = direct_policy.get('ytdlp_config_file') base_config_file = direct_policy.get('ytdlp_config_file')
@ -2243,8 +2255,7 @@ def run_direct_docker_download_worker(worker_id, policy, state_manager, args, pr
os.path.abspath(host_mount_path): {'bind': container_mount_path, 'mode': 'ro'}, os.path.abspath(host_mount_path): {'bind': container_mount_path, 'mode': 'ro'},
os.path.abspath(host_download_path): {'bind': container_download_path, 'mode': 'rw'} os.path.abspath(host_download_path): {'bind': container_download_path, 'mode': 'rw'}
} }
# The command tells yt-dlp where to find the config file we created. # The command tells yt-dlp exactly where to find the config file we created.
# We still set XDG_CONFIG_HOME for any other config it might look for.
command = ['yt-dlp', '--config-locations', os.path.join(config_dir_container, 'yt-dlp/config')] command = ['yt-dlp', '--config-locations', os.path.join(config_dir_container, 'yt-dlp/config')]
logger.info(f"[Worker {worker_id}] [{profile_name}] Running docker command: {' '.join(shlex.quote(s) for s in command)}") logger.info(f"[Worker {worker_id}] [{profile_name}] Running docker command: {' '.join(shlex.quote(s) for s in command)}")