fix: canlı-tespitini tekrar yt-dlp'ye taşı, temiz ISP proxy ekle
Regex tabanlı HTML çıkarımı IP'den bağımsız olarak tutarsız çıktı: YouTube bazen player verisini (videoDetails) sunucu tarafında hiç göndermiyor. yt-dlp'nin kendi extractor'ı engellenmediği sürece her zaman doğru çözüyor, o yüzden kontrol tekrar yt-dlp'ye taşındı — bu sefer üç katmanı birlikte kullanarak: cookie (oturum), pot-provider (PO-token) ve PROXY_URL (temiz residential/ISP IP — Coolify sunucusunun kendi IP'si bir günlük yoğun testten sonra işaretlendi). Oxylabs ISP Proxies ile doğrulandı (gerçek ISP: CenturyLink), datacenter proxy'lerin aksine. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,11 +1,12 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { prisma } from "@streamclipper/db";
|
||||
import { env } from "./env";
|
||||
import { streamIngestQueue } from "./queues";
|
||||
import { sendTelegramMessage } from "./telegram";
|
||||
import { getCookieHeader } from "./ytdlpCookies";
|
||||
import { ytdlpAntiBotArgs } from "./ytdlpCookies";
|
||||
|
||||
const BROWSER_USER_AGENT =
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* Live detection failures are kept per channel and surfaced via GET /status
|
||||
@@ -14,72 +15,35 @@ const BROWSER_USER_AGENT =
|
||||
export const lastPollErrors = new Map<string, string>();
|
||||
|
||||
/**
|
||||
* Diagnostic snapshot of the last check per channel, kept regardless of
|
||||
* outcome — unlike lastPollErrors (cleared on a clean "not live" result),
|
||||
* this stays populated so a genuinely-live channel that's misreported as
|
||||
* offline can still be debugged via GET /status.
|
||||
*/
|
||||
export const lastPollDebug = new Map<string, string>();
|
||||
|
||||
/**
|
||||
* Two earlier approaches both had real problems:
|
||||
* A few approaches were tried before landing here:
|
||||
* - A naive regex over the channel /live page HTML grabbed the FIRST
|
||||
* "videoId" anywhere on the page — which is one of dozens of unrelated
|
||||
* ids (channel video list, recommendations), not the live broadcast.
|
||||
* Confirmed: captured recordings didn't match the real stream.
|
||||
* - Routing the check through yt-dlp resolved the correct video but is
|
||||
* increasingly blocked by YouTube's bot-check/proof-of-origin-token wall
|
||||
* on datacenter IPs — even a plain --simulate call, no cookies fix this
|
||||
* reliably.
|
||||
*
|
||||
* A plain page fetch was never blocked all day — only yt-dlp invocations
|
||||
* were. The fix is extracting the *right* field instead of avoiding the
|
||||
* fetch: YouTube embeds a single `videoDetails` object holding the id/live
|
||||
* flag/channelId of whichever video the page is actually showing (present
|
||||
* only when a channel really is live — verified empty for an offline
|
||||
* channel). Scoping to that one object (and cross-checking channelId)
|
||||
* avoids both the wrong-video bug and yt-dlp's bot-check entirely.
|
||||
*
|
||||
* The Coolify server is EU-geolocated, where an unauthenticated request
|
||||
* hits YouTube's interactive consent wall (no `videoDetails` on that page,
|
||||
* so a genuinely live channel would misreport as offline) — passing the
|
||||
* same logged-in cookies used for yt-dlp skips it, since consent is already
|
||||
* implied by the account.
|
||||
* "videoId" anywhere on the page — one of dozens of unrelated ids
|
||||
* (channel video list, recommendations), not the live broadcast.
|
||||
* - Scoping to the page's `videoDetails` object fixed that, but turned out
|
||||
* unreliable for a different reason: YouTube doesn't always embed the
|
||||
* player response server-side (confirmed on a clean, unflagged IP) — the
|
||||
* field is sometimes just absent regardless of bot-check status.
|
||||
* - yt-dlp's own extractor doesn't have that inconsistency (it resolves the
|
||||
* live video correctly every time it isn't blocked), so the check is
|
||||
* back on yt-dlp — this time with every anti-bot layer combined:
|
||||
* authenticated cookies, the bgutil PO-token provider, and a clean
|
||||
* residential/ISP proxy (the Coolify server's own IP got flagged from a
|
||||
* day of heavy testing). No `-f` format selector here — resolving
|
||||
* downloadable formats is a separate, heavier check only needed later,
|
||||
* in the actual capture.
|
||||
*/
|
||||
async function findLiveVideoId(channelId: string): Promise<string | null> {
|
||||
const liveUrl = `https://www.youtube.com/channel/${channelId}/live`;
|
||||
|
||||
try {
|
||||
const cookieHeader = await getCookieHeader();
|
||||
const headers: Record<string, string> = { "User-Agent": BROWSER_USER_AGENT };
|
||||
if (cookieHeader) headers["Cookie"] = cookieHeader;
|
||||
|
||||
const res = await fetch(liveUrl, { headers });
|
||||
if (!res.ok) {
|
||||
lastPollErrors.set(channelId, `HTTP ${res.status} fetching ${liveUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await res.text();
|
||||
lastPollErrors.delete(channelId);
|
||||
|
||||
const match = html.match(/"videoDetails":\{"videoId":"([^"]+)"/);
|
||||
const hasConsentForm = /action="https:\/\/consent\.youtube\.com/.test(html);
|
||||
const playabilityMatch = html.match(/"playabilityStatus":\{"status":"([^"]*)"(?:,"reason":"([^"]*)")?/);
|
||||
lastPollDebug.set(
|
||||
channelId,
|
||||
`finalUrl=${res.url} htmlLength=${html.length} cookieSent=${Boolean(cookieHeader)} ` +
|
||||
`hasVideoDetails=${Boolean(match)} hasConsentForm=${hasConsentForm} ` +
|
||||
`playability=${playabilityMatch ? `${playabilityMatch[1]}/${playabilityMatch[2] ?? ""}` : "n/a"}`,
|
||||
const { stdout } = await execFileAsync(
|
||||
"yt-dlp",
|
||||
["--simulate", "--no-warnings", ...(await ytdlpAntiBotArgs()), "--print", "%(id)s", liveUrl],
|
||||
{ timeout: 20_000 },
|
||||
);
|
||||
|
||||
if (!match) return null;
|
||||
|
||||
const window = html.slice(match.index!, match.index! + 600);
|
||||
if (!/"isLive":true/.test(window)) return null;
|
||||
if (!window.includes(`"channelId":"${channelId}"`)) return null;
|
||||
|
||||
return match[1];
|
||||
lastPollErrors.delete(channelId);
|
||||
const videoId = stdout.trim().split("\n")[0];
|
||||
return videoId || null;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
lastPollErrors.set(channelId, message.slice(0, 500));
|
||||
|
||||
Reference in New Issue
Block a user