126 lines
4.4 KiB
Docker
126 lines
4.4 KiB
Docker
|
|
|
|
|
|
FROM apache/airflow:2.10.3
|
|
ENV AIRFLOW_VERSION=2.10.3
|
|
|
|
WORKDIR /app
|
|
|
|
# Install system dependencies
|
|
USER root
|
|
RUN apt-get update && \
|
|
apt-get install -y --no-install-recommends \
|
|
vim \
|
|
mc \
|
|
jq \
|
|
build-essential \
|
|
python3-dev \
|
|
wget \
|
|
tar \
|
|
xz-utils \
|
|
iputils-ping \
|
|
curl \
|
|
traceroute \
|
|
tcpdump \
|
|
unzip \
|
|
git && \
|
|
apt-get clean && \
|
|
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc /usr/share/doc-base
|
|
|
|
# Download and install mc (MinIO client)
|
|
RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && \
|
|
chmod +x /usr/local/bin/mc
|
|
|
|
# Install FFmpeg
|
|
RUN FFMPEG_URL="https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz" && \
|
|
wget -qO /tmp/ffmpeg.tar.xz "$FFMPEG_URL" && \
|
|
mkdir -p /opt/ffmpeg && \
|
|
tar -xf /tmp/ffmpeg.tar.xz -C /opt/ffmpeg --strip-components=1 && \
|
|
ln -sf /opt/ffmpeg/bin/ffmpeg /usr/local/bin/ffmpeg && \
|
|
ln -sf /opt/ffmpeg/bin/ffprobe /usr/local/bin/ffprobe && \
|
|
rm -rf /tmp/ffmpeg.tar.xz
|
|
|
|
# Install yt-dlp from master
|
|
RUN python3 -m pip install -U pip hatchling wheel && \
|
|
python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz"
|
|
|
|
# Install Deno
|
|
RUN curl -fsSL https://github.com/denoland/deno/releases/latest/download/deno-x86_64-unknown-linux-gnu.zip -o deno.zip && \
|
|
unzip deno.zip && mv deno /usr/local/bin/ && rm deno.zip
|
|
|
|
# Install aria2c and gost
|
|
RUN curl -fsSL https://raw.githubusercontent.com/P3TERX/aria2-builder/master/aria2-install.sh | bash
|
|
|
|
# Install gost (direct download of binary)
|
|
RUN wget -q https://github.com/ginuerzh/gost/releases/download/v2.12.0/gost_2.12.0_linux_amd64.tar.gz && \
|
|
tar -xzf gost_2.12.0_linux_amd64.tar.gz -C /usr/local/bin/ && \
|
|
rm gost_2.12.0_linux_amd64.tar.gz
|
|
|
|
# Verify installations
|
|
RUN ffmpeg -version && deno --version && yt-dlp --version && aria2c --version && gost -V
|
|
|
|
# Check if airflow group exists, create it if it doesn't, then ensure proper setup
|
|
RUN if ! getent group airflow > /dev/null 2>&1; then \
|
|
groupadd -g 1001 airflow; \
|
|
fi && \
|
|
# Check if airflow user exists and is in the airflow group
|
|
if id -u airflow > /dev/null 2>&1; then \
|
|
usermod -a -G airflow airflow; \
|
|
else \
|
|
useradd -u 1003 -g 1001 -m -s /bin/bash airflow; \
|
|
fi && \
|
|
chown -R airflow:airflow /app && \
|
|
chmod g+w /app
|
|
|
|
# Install base Airflow dependencies
|
|
# [FIX] Explicitly install a version of botocore compatible with Python 3.12
|
|
# to fix a RecursionError when handling S3 remote logs.
|
|
RUN pip install --no-cache-dir \
|
|
"apache-airflow==${AIRFLOW_VERSION}" \
|
|
apache-airflow-providers-docker \
|
|
apache-airflow-providers-http \
|
|
apache-airflow-providers-amazon \
|
|
"botocore>=1.34.118" \
|
|
psycopg2-binary \
|
|
"gunicorn==20.1.0" \
|
|
"python-ffmpeg==2.0.12" \
|
|
"ffprobe3" \
|
|
"python-dotenv"
|
|
|
|
# Switch to airflow user for package installation
|
|
USER airflow
|
|
|
|
# --- Install the custom yt_ops_services package ---
|
|
# Copy all the necessary source code for the package.
|
|
# The deploy script ensures these files are in the build context.
|
|
COPY --chown=airflow:airflow setup.py ./
|
|
COPY --chown=airflow:airflow VERSION ./
|
|
COPY --chown=airflow:airflow yt_ops_services ./yt_ops_services/
|
|
COPY --chown=airflow:airflow thrift_model ./thrift_model/
|
|
COPY --chown=airflow:airflow pangramia ./pangramia/
|
|
|
|
# Install the package in editable mode. This runs setup.py and installs all dependencies
|
|
# listed in `install_requires`, making the `yt_ops_services` module available everywhere.
|
|
RUN pip install --no-cache-dir -e .
|
|
|
|
# Copy token generator scripts and utils with correct permissions
|
|
# COPY --chown=airflow:airflow generate_tokens_direct.mjs ./
|
|
# COPY --chown=airflow:airflow utils ./utils/
|
|
# COPY --chown=airflow:airflow token_generator ./token_generator/
|
|
|
|
# Create version information files
|
|
RUN ( \
|
|
echo "--- yt-dlp ---" && \
|
|
yt-dlp --version && \
|
|
echo "" && \
|
|
echo "--- deno ---" && \
|
|
deno --version && \
|
|
echo "" && \
|
|
echo "--- ffmpeg ---" && \
|
|
ffmpeg -version | head -n 1 \
|
|
) > VERSION-airflow-latest.txt && \
|
|
cp VERSION-airflow-latest.txt VERSION-airflow-$(date +%Y%m%d-%H%M%S).txt
|
|
|
|
# Expose bgutil plugin to worker path
|
|
ENV PYTHONPATH=/opt/bgutil-ytdlp-pot-provider/plugin:$PYTHONPATH
|