diff --git a/.env.example b/.env.example index d8bca42..baf28ba 100644 --- a/.env.example +++ b/.env.example @@ -32,3 +32,10 @@ FORCE_LIVE_URL= # is auto-set to http://sc_pot_provider:4416; leave blank for local dev # unless you're running the provider yourself. YTDLP_POT_PROVIDER_URL= + +# Residential/ISP proxy (http://user:pass@host:port) for yt-dlp calls. +# Datacenter server IPs get progressively more flagged the more they're +# used for yt-dlp requests — a clean residential/ISP IP avoids that. Use an +# ISP proxy product specifically (e.g. Oxylabs "ISP Proxies"), not a plain +# datacenter proxy — the latter has the same problem as the server itself. +PROXY_URL= diff --git a/apps/api-daemon/src/env.ts b/apps/api-daemon/src/env.ts index 789dfa8..4c76b45 100644 --- a/apps/api-daemon/src/env.ts +++ b/apps/api-daemon/src/env.ts @@ -8,5 +8,6 @@ export const env = { port: Number(process.env.API_DAEMON_PORT ?? 4001), forceLiveUrl: process.env.FORCE_LIVE_URL ?? "", ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "", + proxyUrl: process.env.PROXY_URL ?? "", maxConcurrentCaptures: Number(process.env.MAX_CONCURRENT_CAPTURES ?? 10), }; diff --git a/apps/api-daemon/src/server.ts b/apps/api-daemon/src/server.ts index dd08646..451a6b8 100644 --- a/apps/api-daemon/src/server.ts +++ b/apps/api-daemon/src/server.ts @@ -1,7 +1,7 @@ import express from "express"; import { prisma } from "@streamclipper/db"; import { env } from "./env"; -import { lastPollErrors, lastPollDebug } from "./youtubePolling"; +import { lastPollErrors } from "./youtubePolling"; import { lastCaptureDebug } from "./capture/streamIngest"; export function startServer() { @@ -29,7 +29,6 @@ export function startServer() { recording: c.sessions.length > 0, activeSessionId: c.sessions[0]?.id ?? null, lastPollError: lastPollErrors.get(c.channelId) ?? null, - lastPollDebug: lastPollDebug.get(c.channelId) ?? null, })), }); }); diff --git a/apps/api-daemon/src/youtubePolling.ts b/apps/api-daemon/src/youtubePolling.ts index 2e471b1..2c33970 100644 --- a/apps/api-daemon/src/youtubePolling.ts +++ b/apps/api-daemon/src/youtubePolling.ts @@ -1,11 +1,12 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import { prisma } from "@streamclipper/db"; import { env } from "./env"; import { streamIngestQueue } from "./queues"; import { sendTelegramMessage } from "./telegram"; -import { getCookieHeader } from "./ytdlpCookies"; +import { ytdlpAntiBotArgs } from "./ytdlpCookies"; -const BROWSER_USER_AGENT = - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36"; +const execFileAsync = promisify(execFile); /** * Live detection failures are kept per channel and surfaced via GET /status @@ -14,72 +15,35 @@ const BROWSER_USER_AGENT = export const lastPollErrors = new Map(); /** - * Diagnostic snapshot of the last check per channel, kept regardless of - * outcome — unlike lastPollErrors (cleared on a clean "not live" result), - * this stays populated so a genuinely-live channel that's misreported as - * offline can still be debugged via GET /status. - */ -export const lastPollDebug = new Map(); - -/** - * Two earlier approaches both had real problems: + * A few approaches were tried before landing here: * - A naive regex over the channel /live page HTML grabbed the FIRST - * "videoId" anywhere on the page — which is one of dozens of unrelated - * ids (channel video list, recommendations), not the live broadcast. - * Confirmed: captured recordings didn't match the real stream. - * - Routing the check through yt-dlp resolved the correct video but is - * increasingly blocked by YouTube's bot-check/proof-of-origin-token wall - * on datacenter IPs — even a plain --simulate call, no cookies fix this - * reliably. - * - * A plain page fetch was never blocked all day — only yt-dlp invocations - * were. The fix is extracting the *right* field instead of avoiding the - * fetch: YouTube embeds a single `videoDetails` object holding the id/live - * flag/channelId of whichever video the page is actually showing (present - * only when a channel really is live — verified empty for an offline - * channel). Scoping to that one object (and cross-checking channelId) - * avoids both the wrong-video bug and yt-dlp's bot-check entirely. - * - * The Coolify server is EU-geolocated, where an unauthenticated request - * hits YouTube's interactive consent wall (no `videoDetails` on that page, - * so a genuinely live channel would misreport as offline) — passing the - * same logged-in cookies used for yt-dlp skips it, since consent is already - * implied by the account. + * "videoId" anywhere on the page — one of dozens of unrelated ids + * (channel video list, recommendations), not the live broadcast. + * - Scoping to the page's `videoDetails` object fixed that, but turned out + * unreliable for a different reason: YouTube doesn't always embed the + * player response server-side (confirmed on a clean, unflagged IP) — the + * field is sometimes just absent regardless of bot-check status. + * - yt-dlp's own extractor doesn't have that inconsistency (it resolves the + * live video correctly every time it isn't blocked), so the check is + * back on yt-dlp — this time with every anti-bot layer combined: + * authenticated cookies, the bgutil PO-token provider, and a clean + * residential/ISP proxy (the Coolify server's own IP got flagged from a + * day of heavy testing). No `-f` format selector here — resolving + * downloadable formats is a separate, heavier check only needed later, + * in the actual capture. */ async function findLiveVideoId(channelId: string): Promise { const liveUrl = `https://www.youtube.com/channel/${channelId}/live`; try { - const cookieHeader = await getCookieHeader(); - const headers: Record = { "User-Agent": BROWSER_USER_AGENT }; - if (cookieHeader) headers["Cookie"] = cookieHeader; - - const res = await fetch(liveUrl, { headers }); - if (!res.ok) { - lastPollErrors.set(channelId, `HTTP ${res.status} fetching ${liveUrl}`); - return null; - } - - const html = await res.text(); - lastPollErrors.delete(channelId); - - const match = html.match(/"videoDetails":\{"videoId":"([^"]+)"/); - const hasConsentForm = /action="https:\/\/consent\.youtube\.com/.test(html); - const playabilityMatch = html.match(/"playabilityStatus":\{"status":"([^"]*)"(?:,"reason":"([^"]*)")?/); - lastPollDebug.set( - channelId, - `finalUrl=${res.url} htmlLength=${html.length} cookieSent=${Boolean(cookieHeader)} ` + - `hasVideoDetails=${Boolean(match)} hasConsentForm=${hasConsentForm} ` + - `playability=${playabilityMatch ? `${playabilityMatch[1]}/${playabilityMatch[2] ?? ""}` : "n/a"}`, + const { stdout } = await execFileAsync( + "yt-dlp", + ["--simulate", "--no-warnings", ...(await ytdlpAntiBotArgs()), "--print", "%(id)s", liveUrl], + { timeout: 20_000 }, ); - - if (!match) return null; - - const window = html.slice(match.index!, match.index! + 600); - if (!/"isLive":true/.test(window)) return null; - if (!window.includes(`"channelId":"${channelId}"`)) return null; - - return match[1]; + lastPollErrors.delete(channelId); + const videoId = stdout.trim().split("\n")[0]; + return videoId || null; } catch (err) { const message = err instanceof Error ? err.message : String(err); lastPollErrors.set(channelId, message.slice(0, 500)); diff --git a/apps/api-daemon/src/ytdlpCookies.ts b/apps/api-daemon/src/ytdlpCookies.ts index 0b728b7..7e64526 100644 --- a/apps/api-daemon/src/ytdlpCookies.ts +++ b/apps/api-daemon/src/ytdlpCookies.ts @@ -6,11 +6,14 @@ const COOKIES_PATH = "/tmp/yt-cookies.txt"; const COOKIES_SETTING_KEY = "ytdlp_cookies"; /** - * YouTube's anti-bot layer against datacenter IPs has two parts, both - * needed together: a "Sign in to confirm you're not a bot" wall (avoided - * with an authenticated session's cookies) and a "The page needs to be - * reloaded" proof-of-origin token check (avoided by querying the - * bgutil-ytdlp-pot-provider sidecar — see docker-compose.yml sc_pot_provider). + * YouTube's anti-bot layer against yt-dlp has three parts, all needed + * together: a "Sign in to confirm you're not a bot" wall (avoided with an + * authenticated session's cookies), a "The page needs to be reloaded" + * proof-of-origin token check (avoided by querying the bgutil-ytdlp-pot- + * provider sidecar — see docker-compose.yml sc_pot_provider), and — once the + * Coolify server's own IP had a day of heavy testing behind it — outright + * blocking tied to that IP's reputation specifically (avoided by routing + * through a clean residential/ISP proxy, PROXY_URL). * * Cookies rotate/expire over hours, so they're stored in the DB (updated * from the panel's Settings page — see apps/frontend/app/settings) rather @@ -31,27 +34,9 @@ export async function ytdlpAntiBotArgs(): Promise { args.push("--extractor-args", `youtubepot-bgutilhttp:base_url=${env.ytdlpPotProviderUrl}`); } + if (env.proxyUrl) { + args.push("--proxy", env.proxyUrl); + } + return args; } - -/** - * Same cookies, formatted as a `Cookie:` header for plain fetch() calls - * (youtubePolling.ts's live-check). Logged-in cookies also skip YouTube's - * interactive EU consent wall, which an unauthenticated fetch from an - * EU-geolocated server IP otherwise hits — that page has no `videoDetails`, - * so the check would misreport a genuinely live channel as offline. - */ -export async function getCookieHeader(): Promise { - const setting = await prisma.appSetting.findUnique({ where: { key: COOKIES_SETTING_KEY } }); - if (!setting?.value) return null; - - const pairs = setting.value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith("#")) - .map((line) => line.split("\t")) - .filter((fields) => fields.length >= 7) - .map(([, , , , , name, value]) => `${name}=${value}`); - - return pairs.length > 0 ? pairs.join("; ") : null; -} diff --git a/docker-compose.yml b/docker-compose.yml index f74e924..5775651 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-} YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416 MAX_CONCURRENT_CAPTURES: ${MAX_CONCURRENT_CAPTURES:-10} + PROXY_URL: ${PROXY_URL:-} volumes: - shared-media:/shared-media ports: