fix: canlı-tespitini tekrar yt-dlp'ye taşı, temiz ISP proxy ekle

Regex tabanlı HTML çıkarımı IP'den bağımsız olarak tutarsız çıktı:
YouTube bazen player verisini (videoDetails) sunucu tarafında hiç
göndermiyor. yt-dlp'nin kendi extractor'ı engellenmediği sürece her
zaman doğru çözüyor, o yüzden kontrol tekrar yt-dlp'ye taşındı — bu
sefer üç katmanı birlikte kullanarak: cookie (oturum), pot-provider
(PO-token) ve PROXY_URL (temiz residential/ISP IP — Coolify
sunucusunun kendi IP'si bir günlük yoğun testten sonra işaretlendi).
Oxylabs ISP Proxies ile doğrulandı (gerçek ISP: CenturyLink), datacenter
proxy'lerin aksine.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

.env.example

@@ -32,3 +32,10 @@ FORCE_LIVE_URL=
 # is auto-set to http://sc_pot_provider:4416; leave blank for local dev
 # unless you're running the provider yourself.
 YTDLP_POT_PROVIDER_URL=
+
+# Residential/ISP proxy (http://user:pass@host:port) for yt-dlp calls.
+# Datacenter server IPs get progressively more flagged the more they're
+# used for yt-dlp requests — a clean residential/ISP IP avoids that. Use an
+# ISP proxy product specifically (e.g. Oxylabs "ISP Proxies"), not a plain
+# datacenter proxy — the latter has the same problem as the server itself.
+PROXY_URL=

apps/api-daemon/src/env.ts

@@ -8,5 +8,6 @@ export const env = {
   port: Number(process.env.API_DAEMON_PORT ?? 4001),
   forceLiveUrl: process.env.FORCE_LIVE_URL ?? "",
   ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "",
+  proxyUrl: process.env.PROXY_URL ?? "",
   maxConcurrentCaptures: Number(process.env.MAX_CONCURRENT_CAPTURES ?? 10),
 };

apps/api-daemon/src/server.ts

@@ -1,7 +1,7 @@
 import express from "express";
 import { prisma } from "@streamclipper/db";
 import { env } from "./env";
-import { lastPollErrors, lastPollDebug } from "./youtubePolling";
+import { lastPollErrors } from "./youtubePolling";
 import { lastCaptureDebug } from "./capture/streamIngest";
 
 export function startServer() {
@@ -29,7 +29,6 @@ export function startServer() {
         recording: c.sessions.length > 0,
         activeSessionId: c.sessions[0]?.id ?? null,
         lastPollError: lastPollErrors.get(c.channelId) ?? null,
-        lastPollDebug: lastPollDebug.get(c.channelId) ?? null,
       })),
     });
   });

apps/api-daemon/src/youtubePolling.ts

@@ -1,11 +1,12 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
 import { prisma } from "@streamclipper/db";
 import { env } from "./env";
 import { streamIngestQueue } from "./queues";
 import { sendTelegramMessage } from "./telegram";
-import { getCookieHeader } from "./ytdlpCookies";
+import { ytdlpAntiBotArgs } from "./ytdlpCookies";
 
-const BROWSER_USER_AGENT =
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";
+const execFileAsync = promisify(execFile);
 
 /**
  * Live detection failures are kept per channel and surfaced via GET /status
@@ -14,72 +15,35 @@ const BROWSER_USER_AGENT =
 export const lastPollErrors = new Map<string, string>();
 
 /**
- * Diagnostic snapshot of the last check per channel, kept regardless of
- * outcome — unlike lastPollErrors (cleared on a clean "not live" result),
- * this stays populated so a genuinely-live channel that's misreported as
- * offline can still be debugged via GET /status.
- */
-export const lastPollDebug = new Map<string, string>();
-
-/**
- * Two earlier approaches both had real problems:
+ * A few approaches were tried before landing here:
  *  - A naive regex over the channel /live page HTML grabbed the FIRST
- *    "videoId" anywhere on the page — which is one of dozens of unrelated
- *    ids (channel video list, recommendations), not the live broadcast.
- *    Confirmed: captured recordings didn't match the real stream.
- *  - Routing the check through yt-dlp resolved the correct video but is
- *    increasingly blocked by YouTube's bot-check/proof-of-origin-token wall
- *    on datacenter IPs — even a plain --simulate call, no cookies fix this
- *    reliably.
- *
- * A plain page fetch was never blocked all day — only yt-dlp invocations
- * were. The fix is extracting the *right* field instead of avoiding the
- * fetch: YouTube embeds a single `videoDetails` object holding the id/live
- * flag/channelId of whichever video the page is actually showing (present
- * only when a channel really is live — verified empty for an offline
- * channel). Scoping to that one object (and cross-checking channelId)
- * avoids both the wrong-video bug and yt-dlp's bot-check entirely.
- *
- * The Coolify server is EU-geolocated, where an unauthenticated request
- * hits YouTube's interactive consent wall (no `videoDetails` on that page,
- * so a genuinely live channel would misreport as offline) — passing the
- * same logged-in cookies used for yt-dlp skips it, since consent is already
- * implied by the account.
+ *    "videoId" anywhere on the page — one of dozens of unrelated ids
+ *    (channel video list, recommendations), not the live broadcast.
+ *  - Scoping to the page's `videoDetails` object fixed that, but turned out
+ *    unreliable for a different reason: YouTube doesn't always embed the
+ *    player response server-side (confirmed on a clean, unflagged IP) — the
+ *    field is sometimes just absent regardless of bot-check status.
+ *  - yt-dlp's own extractor doesn't have that inconsistency (it resolves the
+ *    live video correctly every time it isn't blocked), so the check is
+ *    back on yt-dlp — this time with every anti-bot layer combined:
+ *    authenticated cookies, the bgutil PO-token provider, and a clean
+ *    residential/ISP proxy (the Coolify server's own IP got flagged from a
+ *    day of heavy testing). No `-f` format selector here — resolving
+ *    downloadable formats is a separate, heavier check only needed later,
+ *    in the actual capture.
  */
 async function findLiveVideoId(channelId: string): Promise<string | null> {
   const liveUrl = `https://www.youtube.com/channel/${channelId}/live`;
 
   try {
-    const cookieHeader = await getCookieHeader();
-    const headers: Record<string, string> = { "User-Agent": BROWSER_USER_AGENT };
-    if (cookieHeader) headers["Cookie"] = cookieHeader;
-
-    const res = await fetch(liveUrl, { headers });
-    if (!res.ok) {
-      lastPollErrors.set(channelId, `HTTP ${res.status} fetching ${liveUrl}`);
-      return null;
-    }
-
-    const html = await res.text();
-    lastPollErrors.delete(channelId);
-
-    const match = html.match(/"videoDetails":\{"videoId":"([^"]+)"/);
-    const hasConsentForm = /action="https:\/\/consent\.youtube\.com/.test(html);
-    const playabilityMatch = html.match(/"playabilityStatus":\{"status":"([^"]*)"(?:,"reason":"([^"]*)")?/);
-    lastPollDebug.set(
-      channelId,
-      `finalUrl=${res.url} htmlLength=${html.length} cookieSent=${Boolean(cookieHeader)} ` +
-        `hasVideoDetails=${Boolean(match)} hasConsentForm=${hasConsentForm} ` +
-        `playability=${playabilityMatch ? `${playabilityMatch[1]}/${playabilityMatch[2] ?? ""}` : "n/a"}`,
+    const { stdout } = await execFileAsync(
+      "yt-dlp",
+      ["--simulate", "--no-warnings", ...(await ytdlpAntiBotArgs()), "--print", "%(id)s", liveUrl],
+      { timeout: 20_000 },
     );
-
-    if (!match) return null;
-
-    const window = html.slice(match.index!, match.index! + 600);
-    if (!/"isLive":true/.test(window)) return null;
-    if (!window.includes(`"channelId":"${channelId}"`)) return null;
-
-    return match[1];
+    lastPollErrors.delete(channelId);
+    const videoId = stdout.trim().split("\n")[0];
+    return videoId || null;
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
     lastPollErrors.set(channelId, message.slice(0, 500));

apps/api-daemon/src/ytdlpCookies.ts

@@ -6,11 +6,14 @@ const COOKIES_PATH = "/tmp/yt-cookies.txt";
 const COOKIES_SETTING_KEY = "ytdlp_cookies";
 
 /**
- * YouTube's anti-bot layer against datacenter IPs has two parts, both
- * needed together: a "Sign in to confirm you're not a bot" wall (avoided
- * with an authenticated session's cookies) and a "The page needs to be
- * reloaded" proof-of-origin token check (avoided by querying the
- * bgutil-ytdlp-pot-provider sidecar — see docker-compose.yml sc_pot_provider).
+ * YouTube's anti-bot layer against yt-dlp has three parts, all needed
+ * together: a "Sign in to confirm you're not a bot" wall (avoided with an
+ * authenticated session's cookies), a "The page needs to be reloaded"
+ * proof-of-origin token check (avoided by querying the bgutil-ytdlp-pot-
+ * provider sidecar — see docker-compose.yml sc_pot_provider), and — once the
+ * Coolify server's own IP had a day of heavy testing behind it — outright
+ * blocking tied to that IP's reputation specifically (avoided by routing
+ * through a clean residential/ISP proxy, PROXY_URL).
  *
  * Cookies rotate/expire over hours, so they're stored in the DB (updated
  * from the panel's Settings page — see apps/frontend/app/settings) rather
@@ -31,27 +34,9 @@ export async function ytdlpAntiBotArgs(): Promise<string[]> {
     args.push("--extractor-args", `youtubepot-bgutilhttp:base_url=${env.ytdlpPotProviderUrl}`);
   }
 
+  if (env.proxyUrl) {
+    args.push("--proxy", env.proxyUrl);
+  }
+
   return args;
 }
-
-/**
- * Same cookies, formatted as a `Cookie:` header for plain fetch() calls
- * (youtubePolling.ts's live-check). Logged-in cookies also skip YouTube's
- * interactive EU consent wall, which an unauthenticated fetch from an
- * EU-geolocated server IP otherwise hits — that page has no `videoDetails`,
- * so the check would misreport a genuinely live channel as offline.
- */
-export async function getCookieHeader(): Promise<string | null> {
-  const setting = await prisma.appSetting.findUnique({ where: { key: COOKIES_SETTING_KEY } });
-  if (!setting?.value) return null;
-
-  const pairs = setting.value
-    .split("\n")
-    .map((line) => line.trim())
-    .filter((line) => line && !line.startsWith("#"))
-    .map((line) => line.split("\t"))
-    .filter((fields) => fields.length >= 7)
-    .map(([, , , , , name, value]) => `${name}=${value}`);
-
-  return pairs.length > 0 ? pairs.join("; ") : null;
-}

docker-compose.yml

@@ -40,6 +40,7 @@ services:
       TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
       YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416
       MAX_CONCURRENT_CAPTURES: ${MAX_CONCURRENT_CAPTURES:-10}
+      PROXY_URL: ${PROXY_URL:-}
     volumes:
       - shared-media:/shared-media
     ports: