perf: her kanalı Oxylabs ISP havuzundaki ayrı bir statik IP'ye sabitle
Dashboard'dan doğrulandı: isp.oxylabs.io:8001-8010 her biri farklı, statik (rotasyon yapmayan) bir IP'ye karşılık geliyor. Ama PROXY_URL'in portu hep sabitti — 7-8 kanalın tüm poll+capture trafiği tek bir IP'den (8001) geçiyordu. Artık her kanal, DB id'sinden deterministik bir hash ile havuzdaki 10 porttan birine sabitleniyor (aynı kanal hep aynı IP'yi kullanıyor, farklı kanallar havuza yayılıyor) — hem YouTube'un tek IP üzerindeki yük algısını azaltıyor hem de bant genişliğini paylaştırıyor. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -96,7 +96,7 @@ async function processCompletedSegments(
|
||||
}
|
||||
|
||||
async function runCapture(job: Job<StreamIngestJob>): Promise<void> {
|
||||
const { sessionId, youtubeUrl, segmentTimeSec } = job.data;
|
||||
const { sessionId, youtubeUrl, segmentTimeSec, channelDbId } = job.data;
|
||||
const outDir = path.join(env.sharedMediaRoot, "raw", sessionId);
|
||||
await mkdir(outDir, { recursive: true });
|
||||
|
||||
@@ -108,7 +108,7 @@ async function runCapture(job: Job<StreamIngestJob>): Promise<void> {
|
||||
"yt-dlp",
|
||||
[
|
||||
"--js-runtimes", "node",
|
||||
...(await ytdlpAntiBotArgs()),
|
||||
...(await ytdlpAntiBotArgs(channelDbId)),
|
||||
"-f", "bestvideo+bestaudio/best", "-o", "-", youtubeUrl,
|
||||
],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||
|
||||
@@ -76,7 +76,7 @@ async function findLiveVideoId(channelId: string, channelName: string, channelDb
|
||||
"--no-warnings",
|
||||
"--js-runtimes",
|
||||
"node",
|
||||
...(await ytdlpAntiBotArgs()),
|
||||
...(await ytdlpAntiBotArgs(channelDbId)),
|
||||
"--print",
|
||||
"%(id)s",
|
||||
liveUrl,
|
||||
|
||||
@@ -5,6 +5,31 @@ import { env } from "./env";
|
||||
const COOKIES_PATH = "/tmp/yt-cookies.txt";
|
||||
const COOKIES_SETTING_KEY = "ytdlp_cookies";
|
||||
|
||||
// Oxylabs' static ISP pool: one dedicated (non-rotating) IP per port,
|
||||
// confirmed against the dashboard's Proxy list (isp.oxylabs.io:8001..8010 ->
|
||||
// 10 distinct IPs). PROXY_URL's own port is the base; funneling every
|
||||
// channel through that single port meant 7-8 simultaneous channels' worth
|
||||
// of YouTube-facing traffic all looked like it came from one IP.
|
||||
const PROXY_PORT_POOL_SIZE = 10;
|
||||
|
||||
function simpleHash(input: string): number {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash = (hash * 31 + input.charCodeAt(i)) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
/** Deterministically pins a channel to one of the pool's ports — same channel always gets the same IP (so it isn't mid-session IP-mismatched against a live URL signed for a different one), different channels spread across the pool. */
|
||||
function proxyUrlForChannel(baseProxyUrl: string, channelKey: string): string {
|
||||
const match = baseProxyUrl.match(/^(.*:)(\d+)$/);
|
||||
if (!match) return baseProxyUrl;
|
||||
const [, prefix, basePortStr] = match;
|
||||
const basePort = Number(basePortStr);
|
||||
const offset = simpleHash(channelKey) % PROXY_PORT_POOL_SIZE;
|
||||
return `${prefix}${basePort + offset}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* YouTube's anti-bot layer against yt-dlp has three parts, all needed
|
||||
* together: a "Sign in to confirm you're not a bot" wall (avoided with an
|
||||
@@ -21,7 +46,7 @@ const COOKIES_SETTING_KEY = "ytdlp_cookies";
|
||||
* current value and rewrites the temp file, so a panel update takes effect
|
||||
* on the very next yt-dlp invocation without a redeploy.
|
||||
*/
|
||||
export async function ytdlpAntiBotArgs(): Promise<string[]> {
|
||||
export async function ytdlpAntiBotArgs(channelKey: string): Promise<string[]> {
|
||||
const args: string[] = [];
|
||||
|
||||
const setting = await prisma.appSetting.findUnique({ where: { key: COOKIES_SETTING_KEY } });
|
||||
@@ -42,7 +67,7 @@ export async function ytdlpAntiBotArgs(): Promise<string[]> {
|
||||
args.push("--extractor-args", "youtube:player_client=web,mweb,android");
|
||||
|
||||
if (env.proxyUrl) {
|
||||
args.push("--proxy", env.proxyUrl);
|
||||
args.push("--proxy", proxyUrlForChannel(env.proxyUrl, channelKey));
|
||||
}
|
||||
|
||||
return args;
|
||||
|
||||
Reference in New Issue
Block a user