import { writeFileSync } from "node:fs"; import { prisma } from "@streamclipper/db"; import { env } from "./env"; const LEGACY_COOKIES_PATH = "/tmp/yt-cookies.txt"; const LEGACY_COOKIES_SETTING_KEY = "ytdlp_cookies"; // Oxylabs' static ISP pool: one dedicated (non-rotating) IP per port, // confirmed against the dashboard's Proxy list (isp.oxylabs.io:8001..8010 -> // 10 distinct IPs). PROXY_URL's own port is the base; funneling every // channel through that single port meant 7-8 simultaneous channels' worth // of YouTube-facing traffic all looked like it came from one IP. const PROXY_PORT_POOL_SIZE = 10; function simpleHash(input: string): number { let hash = 0; for (let i = 0; i < input.length; i++) { hash = (hash * 31 + input.charCodeAt(i)) >>> 0; } return hash; } /** Deterministically pins a channel to one of the pool's ports — same channel always gets the same IP (so it isn't mid-session IP-mismatched against a live URL signed for a different one), different channels spread across the pool. */ function proxyUrlForChannel(baseProxyUrl: string, channelKey: string): string { const match = baseProxyUrl.match(/^(.*:)(\d+)$/); if (!match) return baseProxyUrl; const [, prefix, basePortStr] = match; const basePort = Number(basePortStr); const offset = simpleHash(channelKey) % PROXY_PORT_POOL_SIZE; return `${prefix}${basePort + offset}`; } /** * Picks which cookie profile a channel uses (same deterministic-hash * pinning as proxyUrlForChannel — same channel always gets the same * account, different channels spread across the pool) and writes it to a * profile-specific temp file. Falls back to the old single global * ytdlp_cookies AppSetting if no profiles have been added yet, so nothing * breaks before the panel's Settings page is used to add accounts. * * A single shared cookie file was a real bottleneck once several channels * were active at once: it isn't just proxy IP reputation that YouTube can * flag, it's the *account* — one Google session making requests that look * like it's scanning many unrelated channels trips its own abuse signal, * independent of which IP each request came from. Splitting across * separate throwaway accounts dilutes that per-account signal the same way * the proxy pool dilutes per-IP signal. * * Profile-specific file paths (not one shared /tmp/yt-cookies.txt) matter * once multiple channels can be using *different* profiles concurrently — * a shared path would let one channel's capture process read a file another * channel's poll check just overwrote with a different account's cookies. */ async function cookiesFilePathForChannel(channelKey: string): Promise { const profiles = await prisma.cookieProfile.findMany({ orderBy: { createdAt: "asc" } }); if (profiles.length === 0) { const legacy = await prisma.appSetting.findUnique({ where: { key: LEGACY_COOKIES_SETTING_KEY } }); if (!legacy?.value) return null; writeFileSync(LEGACY_COOKIES_PATH, legacy.value, { mode: 0o600 }); return LEGACY_COOKIES_PATH; } const profile = profiles[simpleHash(channelKey) % profiles.length]; const path = `/tmp/yt-cookies-${profile.id}.txt`; writeFileSync(path, profile.value, { mode: 0o600 }); return path; } /** * YouTube's anti-bot layer against yt-dlp has three parts, all needed * together: a "Sign in to confirm you're not a bot" wall (avoided with an * authenticated session's cookies), a "The page needs to be reloaded" * proof-of-origin token check (avoided by querying the bgutil-ytdlp-pot- * provider sidecar — see docker-compose.yml sc_pot_provider), and — once the * Coolify server's own IP had a day of heavy testing behind it — outright * blocking tied to that IP's reputation specifically (avoided by routing * through a clean residential/ISP proxy, PROXY_URL). * * Cookies rotate/expire over hours, so they're stored in the DB (updated * from the panel's Settings page — see apps/frontend/app/settings) rather * than baked in at container start from an env var. Every call re-reads the * current value and rewrites the temp file, so a panel update takes effect * on the very next yt-dlp invocation without a redeploy. */ export async function ytdlpAntiBotArgs( channelKey: string, { includeAndroidClient = true }: { includeAndroidClient?: boolean } = {}, ): Promise { const args: string[] = []; const cookiesPath = await cookiesFilePathForChannel(channelKey); if (cookiesPath) { args.push("--cookies", cookiesPath); } if (env.ytdlpPotProviderUrl) { args.push("--extractor-args", `youtubepot-bgutilhttp:base_url=${env.ytdlpPotProviderUrl}`); } // Büyük/popüler kanallarda (ör. NASA, çok izleyicili yayınlar) sade "web" // client'ı cookie+PO-token kombinasyonuna rağmen "Sign in to confirm // you're not a bot" ile tıkanabiliyor — küçük kanallarda aynı kombinasyon // sorunsuz çalışıyor. mweb/android'i cookie-uyumlu fallback olarak // ekleyip yt-dlp'nin ilk başarılı client'ı kullanmasını sağlıyoruz. // // Ama "android" client'ı yt-dlp'nin kendi GitHub issue'larında birden // fazla kez videoplayback/segment isteklerinde 403 Forbidden'a sebep // olarak raporlanmış — sade metadata/tespit isteğinde (--simulate, format // çözümlemesi yok) risk taşımıyor ama gerçek segment indirimi sırasında // (asıl capture) bu 403'lerin kaynağı olabilir. O yüzden capture çağrısı // android'i devre dışı bırakıyor, sadece tespit çağrısı kullanıyor. const clients = includeAndroidClient ? "web,mweb,android" : "web,mweb"; args.push("--extractor-args", `youtube:player_client=${clients}`); if (env.proxyUrl) { args.push("--proxy", proxyUrlForChannel(env.proxyUrl, channelKey)); } return args; }