FROM apache/airflow:2.10.3 ENV AIRFLOW_VERSION=2.10.3 WORKDIR /app # Install system dependencies USER root RUN apt-get update && \ apt-get install -y --no-install-recommends \ vim \ mc \ jq \ build-essential \ python3-dev \ wget \ tar \ xz-utils \ iputils-ping \ curl \ traceroute \ tcpdump \ unzip \ git && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc /usr/share/doc-base # Download and install mc (MinIO client) RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && \ chmod +x /usr/local/bin/mc # Install FFmpeg RUN FFMPEG_URL="https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz" && \ wget -qO /tmp/ffmpeg.tar.xz "$FFMPEG_URL" && \ mkdir -p /opt/ffmpeg && \ tar -xf /tmp/ffmpeg.tar.xz -C /opt/ffmpeg --strip-components=1 && \ ln -sf /opt/ffmpeg/bin/ffmpeg /usr/local/bin/ffmpeg && \ ln -sf /opt/ffmpeg/bin/ffprobe /usr/local/bin/ffprobe && \ rm -rf /tmp/ffmpeg.tar.xz # Install yt-dlp from master RUN python3 -m pip install -U pip hatchling wheel && \ python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz" # Install Deno RUN curl -fsSL https://github.com/denoland/deno/releases/latest/download/deno-x86_64-unknown-linux-gnu.zip -o deno.zip && \ unzip deno.zip && mv deno /usr/local/bin/ && rm deno.zip # Install aria2c and gost RUN curl -fsSL https://raw.githubusercontent.com/P3TERX/aria2-builder/master/aria2-install.sh | bash # Install gost (direct download of binary) RUN wget -q https://github.com/ginuerzh/gost/releases/download/v2.12.0/gost_2.12.0_linux_amd64.tar.gz && \ tar -xzf gost_2.12.0_linux_amd64.tar.gz -C /usr/local/bin/ && \ rm gost_2.12.0_linux_amd64.tar.gz # Verify installations RUN ffmpeg -version && deno --version && yt-dlp --version && aria2c --version && gost -V # Check if airflow group exists, create it if it doesn't, then ensure proper setup RUN if ! getent group airflow > /dev/null 2>&1; then \ groupadd -g 1001 airflow; \ fi && \ # Check if airflow user exists and is in the airflow group if id -u airflow > /dev/null 2>&1; then \ usermod -a -G airflow airflow; \ else \ useradd -u 1003 -g 1001 -m -s /bin/bash airflow; \ fi && \ chown -R airflow:airflow /app && \ chmod g+w /app # Install base Airflow dependencies # [FIX] Explicitly install a version of botocore compatible with Python 3.12 # to fix a RecursionError when handling S3 remote logs. RUN pip install --no-cache-dir \ "apache-airflow==${AIRFLOW_VERSION}" \ apache-airflow-providers-docker \ apache-airflow-providers-http \ apache-airflow-providers-amazon \ "botocore>=1.34.118" \ psycopg2-binary \ "gunicorn==20.1.0" \ "python-ffmpeg==2.0.12" \ "ffprobe3" \ "python-dotenv" # Switch to airflow user for package installation USER airflow # --- Install the custom yt_ops_services package --- # Copy all the necessary source code for the package. # The deploy script ensures these files are in the build context. COPY --chown=airflow:airflow setup.py ./ COPY --chown=airflow:airflow VERSION ./ COPY --chown=airflow:airflow yt_ops_services ./yt_ops_services/ COPY --chown=airflow:airflow thrift_model ./thrift_model/ COPY --chown=airflow:airflow pangramia ./pangramia/ # Install the package in editable mode. This runs setup.py and installs all dependencies # listed in `install_requires`, making the `yt_ops_services` module available everywhere. RUN pip install --no-cache-dir -e . # Copy token generator scripts and utils with correct permissions # COPY --chown=airflow:airflow generate_tokens_direct.mjs ./ # COPY --chown=airflow:airflow utils ./utils/ # COPY --chown=airflow:airflow token_generator ./token_generator/ # Create version information files RUN ( \ echo "--- yt-dlp ---" && \ yt-dlp --version && \ echo "" && \ echo "--- deno ---" && \ deno --version && \ echo "" && \ echo "--- ffmpeg ---" && \ ffmpeg -version | head -n 1 \ ) > VERSION-airflow-latest.txt && \ cp VERSION-airflow-latest.txt VERSION-airflow-$(date +%Y%m%d-%H%M%S).txt # Expose bgutil plugin to worker path ENV PYTHONPATH=/opt/bgutil-ytdlp-pot-provider/plugin:$PYTHONPATH