fix: bgutil-ytdlp-pot-provider ile PO-token duvarını aş
Cookie bot-check'i çözdü ama "The page needs to be reloaded" hatası kalıcıydı — YouTube artık proof-of-origin token da istiyor. Coolify stack'ine brainicism/bgutil-ytdlp-pot-provider sidecar servisi (sc_pot_provider, port 4416) eklendi, api-daemon imajına Python plugin'i kuruldu, her yt-dlp çağrısına --extractor-args youtubepot-bgutilhttp:base_url=... eklendi. ytdlpCookieArgs -> ytdlpAntiBotArgs olarak birleştirildi. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -26,3 +26,10 @@ FORCE_LIVE_URL=
|
|||||||
# avoid it. Use a throwaway/secondary Google account, not your main one —
|
# avoid it. Use a throwaway/secondary Google account, not your main one —
|
||||||
# this file is a live session credential. Never commit the raw cookies.txt.
|
# this file is a live session credential. Never commit the raw cookies.txt.
|
||||||
YTDLP_COOKIES_B64=
|
YTDLP_COOKIES_B64=
|
||||||
|
|
||||||
|
# URL of a bgutil-ytdlp-pot-provider HTTP server (see docker-compose.yml
|
||||||
|
# sc_pot_provider). Needed alongside cookies to avoid YouTube's "The page
|
||||||
|
# needs to be reloaded" proof-of-origin token check. In docker-compose this
|
||||||
|
# is auto-set to http://sc_pot_provider:4416; leave blank for local dev
|
||||||
|
# unless you're running the provider yourself.
|
||||||
|
YTDLP_POT_PROVIDER_URL=
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ FROM node:22-slim
|
|||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
ffmpeg python3 python3-pip ca-certificates \
|
ffmpeg python3 python3-pip ca-certificates \
|
||||||
&& pip3 install --break-system-packages -U yt-dlp \
|
&& pip3 install --break-system-packages -U yt-dlp bgutil-ytdlp-pot-provider \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN corepack enable
|
RUN corepack enable
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import { prisma } from "@streamclipper/db";
|
|||||||
import { env } from "../env";
|
import { env } from "../env";
|
||||||
import { QUEUE_NAMES, signalDetectionQueue, type StreamIngestJob } from "../queues";
|
import { QUEUE_NAMES, signalDetectionQueue, type StreamIngestJob } from "../queues";
|
||||||
import { redisConnection } from "../redis";
|
import { redisConnection } from "../redis";
|
||||||
import { ytdlpCookieArgs } from "../ytdlpCookies";
|
import { ytdlpAntiBotArgs } from "../ytdlpCookies";
|
||||||
|
|
||||||
const SEGMENT_LIST_POLL_MS = 5_000;
|
const SEGMENT_LIST_POLL_MS = 5_000;
|
||||||
|
|
||||||
@@ -75,7 +75,7 @@ async function runCapture(job: Job<StreamIngestJob>): Promise<void> {
|
|||||||
const ytdlp = spawn(
|
const ytdlp = spawn(
|
||||||
"yt-dlp",
|
"yt-dlp",
|
||||||
[
|
[
|
||||||
...ytdlpCookieArgs(),
|
...ytdlpAntiBotArgs(),
|
||||||
"-f", "bestvideo+bestaudio/best", "-o", "-", youtubeUrl,
|
"-f", "bestvideo+bestaudio/best", "-o", "-", youtubeUrl,
|
||||||
],
|
],
|
||||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||||
|
|||||||
@@ -8,4 +8,5 @@ export const env = {
|
|||||||
port: Number(process.env.API_DAEMON_PORT ?? 4001),
|
port: Number(process.env.API_DAEMON_PORT ?? 4001),
|
||||||
forceLiveUrl: process.env.FORCE_LIVE_URL ?? "",
|
forceLiveUrl: process.env.FORCE_LIVE_URL ?? "",
|
||||||
ytdlpCookiesB64: process.env.YTDLP_COOKIES_B64 ?? "",
|
ytdlpCookiesB64: process.env.YTDLP_COOKIES_B64 ?? "",
|
||||||
|
ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "",
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { prisma } from "@streamclipper/db";
|
|||||||
import { env } from "./env";
|
import { env } from "./env";
|
||||||
import { streamIngestQueue } from "./queues";
|
import { streamIngestQueue } from "./queues";
|
||||||
import { sendTelegramMessage } from "./telegram";
|
import { sendTelegramMessage } from "./telegram";
|
||||||
import { ytdlpCookieArgs } from "./ytdlpCookies";
|
import { ytdlpAntiBotArgs } from "./ytdlpCookies";
|
||||||
|
|
||||||
const execFileAsync = promisify(execFile);
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
@@ -32,7 +32,7 @@ async function findLiveVideoId(channelId: string): Promise<string | null> {
|
|||||||
[
|
[
|
||||||
"--simulate",
|
"--simulate",
|
||||||
"--no-warnings",
|
"--no-warnings",
|
||||||
...ytdlpCookieArgs(),
|
...ytdlpAntiBotArgs(),
|
||||||
"-f", "bestvideo+bestaudio/best",
|
"-f", "bestvideo+bestaudio/best",
|
||||||
"--print", "%(id)s",
|
"--print", "%(id)s",
|
||||||
liveUrl,
|
liveUrl,
|
||||||
|
|||||||
@@ -4,11 +4,13 @@ import { env } from "./env";
|
|||||||
const COOKIES_PATH = "/tmp/yt-cookies.txt";
|
const COOKIES_PATH = "/tmp/yt-cookies.txt";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* YouTube increasingly bot-checks yt-dlp requests from datacenter IPs
|
* YouTube's anti-bot layer against datacenter IPs has two parts, both
|
||||||
* ("Sign in to confirm you're not a bot"), which no --extractor-args
|
* needed together: a "Sign in to confirm you're not a bot" wall (avoided
|
||||||
* combination reliably bypasses. An authenticated session's cookies avoid
|
* with an authenticated session's cookies) and a "The page needs to be
|
||||||
* the check. The cookies file is never committed — it's decoded once at
|
* reloaded" proof-of-origin token check (avoided by querying the
|
||||||
* startup from a Coolify-managed env var into a local temp file.
|
* bgutil-ytdlp-pot-provider sidecar — see docker-compose.yml sc_pot_provider).
|
||||||
|
* The cookies file is never committed — it's decoded once at startup from a
|
||||||
|
* Coolify-managed env var into a local temp file.
|
||||||
*/
|
*/
|
||||||
const cookiesFilePath: string | null = env.ytdlpCookiesB64
|
const cookiesFilePath: string | null = env.ytdlpCookiesB64
|
||||||
? (() => {
|
? (() => {
|
||||||
@@ -17,6 +19,16 @@ const cookiesFilePath: string | null = env.ytdlpCookiesB64
|
|||||||
})()
|
})()
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
export function ytdlpCookieArgs(): string[] {
|
export function ytdlpAntiBotArgs(): string[] {
|
||||||
return cookiesFilePath ? ["--cookies", cookiesFilePath] : [];
|
const args: string[] = [];
|
||||||
|
|
||||||
|
if (cookiesFilePath) {
|
||||||
|
args.push("--cookies", cookiesFilePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (env.ytdlpPotProviderUrl) {
|
||||||
|
args.push("--extractor-args", `youtubepot-bgutilhttp:base_url=${env.ytdlpPotProviderUrl}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return args;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- postgres-data:/var/lib/postgresql/data
|
- postgres-data:/var/lib/postgresql/data
|
||||||
|
|
||||||
|
sc_pot_provider:
|
||||||
|
image: brainicism/bgutil-ytdlp-pot-provider
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
sc_api_daemon:
|
sc_api_daemon:
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
@@ -35,6 +39,7 @@ services:
|
|||||||
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
|
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
|
||||||
TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
|
TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
|
||||||
YTDLP_COOKIES_B64: ${YTDLP_COOKIES_B64:-}
|
YTDLP_COOKIES_B64: ${YTDLP_COOKIES_B64:-}
|
||||||
|
YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416
|
||||||
volumes:
|
volumes:
|
||||||
- shared-media:/shared-media
|
- shared-media:/shared-media
|
||||||
ports:
|
ports:
|
||||||
@@ -42,6 +47,7 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
- sc_redis
|
- sc_redis
|
||||||
- sc_postgres
|
- sc_postgres
|
||||||
|
- sc_pot_provider
|
||||||
|
|
||||||
sc_heavy_worker:
|
sc_heavy_worker:
|
||||||
build:
|
build:
|
||||||
|
|||||||
Reference in New Issue
Block a user