diff --git a/.env.example b/.env.example index 1f549ee..c8aad70 100644 --- a/.env.example +++ b/.env.example @@ -26,3 +26,10 @@ FORCE_LIVE_URL= # avoid it. Use a throwaway/secondary Google account, not your main one — # this file is a live session credential. Never commit the raw cookies.txt. YTDLP_COOKIES_B64= + +# URL of a bgutil-ytdlp-pot-provider HTTP server (see docker-compose.yml +# sc_pot_provider). Needed alongside cookies to avoid YouTube's "The page +# needs to be reloaded" proof-of-origin token check. In docker-compose this +# is auto-set to http://sc_pot_provider:4416; leave blank for local dev +# unless you're running the provider yourself. +YTDLP_POT_PROVIDER_URL= diff --git a/apps/api-daemon/Dockerfile b/apps/api-daemon/Dockerfile index 54008f5..505ebc0 100644 --- a/apps/api-daemon/Dockerfile +++ b/apps/api-daemon/Dockerfile @@ -2,7 +2,7 @@ FROM node:22-slim RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg python3 python3-pip ca-certificates \ - && pip3 install --break-system-packages -U yt-dlp \ + && pip3 install --break-system-packages -U yt-dlp bgutil-ytdlp-pot-provider \ && rm -rf /var/lib/apt/lists/* RUN corepack enable diff --git a/apps/api-daemon/src/capture/streamIngest.ts b/apps/api-daemon/src/capture/streamIngest.ts index 2636d2b..143b5a2 100644 --- a/apps/api-daemon/src/capture/streamIngest.ts +++ b/apps/api-daemon/src/capture/streamIngest.ts @@ -6,7 +6,7 @@ import { prisma } from "@streamclipper/db"; import { env } from "../env"; import { QUEUE_NAMES, signalDetectionQueue, type StreamIngestJob } from "../queues"; import { redisConnection } from "../redis"; -import { ytdlpCookieArgs } from "../ytdlpCookies"; +import { ytdlpAntiBotArgs } from "../ytdlpCookies"; const SEGMENT_LIST_POLL_MS = 5_000; @@ -75,7 +75,7 @@ async function runCapture(job: Job): Promise { const ytdlp = spawn( "yt-dlp", [ - ...ytdlpCookieArgs(), + ...ytdlpAntiBotArgs(), "-f", "bestvideo+bestaudio/best", "-o", "-", youtubeUrl, ], { stdio: ["ignore", "pipe", "pipe"] }, diff --git a/apps/api-daemon/src/env.ts b/apps/api-daemon/src/env.ts index 246c891..4d401f0 100644 --- a/apps/api-daemon/src/env.ts +++ b/apps/api-daemon/src/env.ts @@ -8,4 +8,5 @@ export const env = { port: Number(process.env.API_DAEMON_PORT ?? 4001), forceLiveUrl: process.env.FORCE_LIVE_URL ?? "", ytdlpCookiesB64: process.env.YTDLP_COOKIES_B64 ?? "", + ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "", }; diff --git a/apps/api-daemon/src/youtubePolling.ts b/apps/api-daemon/src/youtubePolling.ts index cb9bc4c..083fbfc 100644 --- a/apps/api-daemon/src/youtubePolling.ts +++ b/apps/api-daemon/src/youtubePolling.ts @@ -4,7 +4,7 @@ import { prisma } from "@streamclipper/db"; import { env } from "./env"; import { streamIngestQueue } from "./queues"; import { sendTelegramMessage } from "./telegram"; -import { ytdlpCookieArgs } from "./ytdlpCookies"; +import { ytdlpAntiBotArgs } from "./ytdlpCookies"; const execFileAsync = promisify(execFile); @@ -32,7 +32,7 @@ async function findLiveVideoId(channelId: string): Promise { [ "--simulate", "--no-warnings", - ...ytdlpCookieArgs(), + ...ytdlpAntiBotArgs(), "-f", "bestvideo+bestaudio/best", "--print", "%(id)s", liveUrl, diff --git a/apps/api-daemon/src/ytdlpCookies.ts b/apps/api-daemon/src/ytdlpCookies.ts index 4314409..e6b203b 100644 --- a/apps/api-daemon/src/ytdlpCookies.ts +++ b/apps/api-daemon/src/ytdlpCookies.ts @@ -4,11 +4,13 @@ import { env } from "./env"; const COOKIES_PATH = "/tmp/yt-cookies.txt"; /** - * YouTube increasingly bot-checks yt-dlp requests from datacenter IPs - * ("Sign in to confirm you're not a bot"), which no --extractor-args - * combination reliably bypasses. An authenticated session's cookies avoid - * the check. The cookies file is never committed — it's decoded once at - * startup from a Coolify-managed env var into a local temp file. + * YouTube's anti-bot layer against datacenter IPs has two parts, both + * needed together: a "Sign in to confirm you're not a bot" wall (avoided + * with an authenticated session's cookies) and a "The page needs to be + * reloaded" proof-of-origin token check (avoided by querying the + * bgutil-ytdlp-pot-provider sidecar — see docker-compose.yml sc_pot_provider). + * The cookies file is never committed — it's decoded once at startup from a + * Coolify-managed env var into a local temp file. */ const cookiesFilePath: string | null = env.ytdlpCookiesB64 ? (() => { @@ -17,6 +19,16 @@ const cookiesFilePath: string | null = env.ytdlpCookiesB64 })() : null; -export function ytdlpCookieArgs(): string[] { - return cookiesFilePath ? ["--cookies", cookiesFilePath] : []; +export function ytdlpAntiBotArgs(): string[] { + const args: string[] = []; + + if (cookiesFilePath) { + args.push("--cookies", cookiesFilePath); + } + + if (env.ytdlpPotProviderUrl) { + args.push("--extractor-args", `youtubepot-bgutilhttp:base_url=${env.ytdlpPotProviderUrl}`); + } + + return args; } diff --git a/docker-compose.yml b/docker-compose.yml index 445d2f2..66841a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,10 @@ services: volumes: - postgres-data:/var/lib/postgresql/data + sc_pot_provider: + image: brainicism/bgutil-ytdlp-pot-provider + restart: unless-stopped + sc_api_daemon: build: context: . @@ -35,6 +39,7 @@ services: TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-} TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-} YTDLP_COOKIES_B64: ${YTDLP_COOKIES_B64:-} + YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416 volumes: - shared-media:/shared-media ports: @@ -42,6 +47,7 @@ services: depends_on: - sc_redis - sc_postgres + - sc_pot_provider sc_heavy_worker: build: