StreamClipper AI Faz 1: ingestion + sinyal analizi + STT + altyapı iskeleti

yt-dlp headless capture (15dk segmentleme), ses peak + chat velocity sinyal tespiti, OpenAI Whisper STT (kelime zaman damgalı), BullMQ/Redis/Postgres altyapısı ve kanal durumu + transkript kütüphanesi gösteren Next.js panel. LLM virality skorlama, render/crop/altyazı ve multi-platform dağıtım bu fazın kapsamı dışında. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-30 14:36:50 +03:00
commit eccc74166a
44 changed files with 3897 additions and 0 deletions
@@ -0,0 +1,15 @@
+FROM node:20-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ffmpeg python3 python3-pip ca-certificates \
+    && pip3 install --break-system-packages -U yt-dlp \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN corepack enable
+
+WORKDIR /app
+COPY . .
+RUN pnpm install
+RUN pnpm --filter @streamclipper/db exec prisma generate
+
+CMD ["pnpm", "--filter", "@streamclipper/api-daemon", "dev"]
@@ -0,0 +1,24 @@
+{
+  "name": "@streamclipper/api-daemon",
+  "private": true,
+  "version": "0.1.0",
+  "type": "commonjs",
+  "scripts": {
+    "dev": "tsx watch src/index.ts",
+    "start": "node dist/index.js",
+    "build": "tsc"
+  },
+  "dependencies": {
+    "@streamclipper/db": "workspace:*",
+    "bullmq": "^5.34.0",
+    "dotenv": "^16.4.5",
+    "express": "^4.21.1",
+    "ioredis": "^5.4.1"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/node": "^22.10.0",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.0"
+  }
+}
@@ -0,0 +1,152 @@
+import { spawn } from "node:child_process";
+import { mkdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { Worker, type Job } from "bullmq";
+import { prisma } from "@streamclipper/db";
+import { env } from "../env";
+import { QUEUE_NAMES, signalDetectionQueue, type StreamIngestJob } from "../queues";
+import { redisConnection } from "../redis";
+
+const SEGMENT_LIST_POLL_MS = 5_000;
+
+interface SegmentListEntry {
+  fileName: string;
+  startSec: number;
+  endSec: number;
+}
+
+function parseSegmentListLines(raw: string): SegmentListEntry[] {
+  return raw
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      const [fileName, startStr, endStr] = line.split(",");
+      return { fileName, startSec: Number(startStr), endSec: Number(endStr) };
+    });
+}
+
+async function processCompletedSegments(
+  entries: SegmentListEntry[],
+  fromIndex: number,
+  outDir: string,
+  sessionId: string,
+): Promise<number> {
+  let processed = fromIndex;
+
+  for (let i = fromIndex; i < entries.length; i++) {
+    const entry = entries[i];
+    const filePath = path.join(outDir, entry.fileName);
+    const duration = Math.round(entry.endSec - entry.startSec);
+
+    const segment = await prisma.rawSegment.create({
+      data: {
+        sessionId,
+        filePath,
+        duration,
+        startedAt: new Date(Date.now() - duration * 1000),
+        status: "PENDING",
+      },
+    });
+
+    await signalDetectionQueue.add("analyze-segment", {
+      rawSegmentId: segment.id,
+      filePath,
+      sessionId,
+    });
+
+    console.log(`[stream-ingest] segment ready: ${entry.fileName} -> raw_segment ${segment.id}`);
+    processed = i + 1;
+  }
+
+  return processed;
+}
+
+async function runCapture(job: Job<StreamIngestJob>): Promise<void> {
+  const { sessionId, youtubeUrl } = job.data;
+  const outDir = path.join(env.sharedMediaRoot, "raw", sessionId);
+  await mkdir(outDir, { recursive: true });
+
+  const segmentListPath = path.join(outDir, "segments.csv");
+
+  console.log(`[stream-ingest] starting capture for session ${sessionId}: ${youtubeUrl}`);
+
+  const ytdlp = spawn("yt-dlp", ["-f", "best", "-o", "-", youtubeUrl], {
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  const ffmpeg = spawn(
+    "ffmpeg",
+    [
+      "-i", "pipe:0",
+      "-c", "copy",
+      "-f", "segment",
+      "-segment_time", String(env.segmentTimeSec),
+      "-reset_timestamps", "1",
+      "-segment_list", segmentListPath,
+      "-segment_list_type", "csv",
+      path.join(outDir, "segment_%03d.mp4"),
+    ],
+    { stdio: ["pipe", "pipe", "pipe"] },
+  );
+
+  ytdlp.stdout.pipe(ffmpeg.stdin);
+  ytdlp.stderr.on("data", (chunk) => console.error(`[yt-dlp:${sessionId}]`, chunk.toString().trim()));
+  ffmpeg.stderr.on("data", () => {
+    /* ffmpeg logs are extremely verbose; only surface on error below */
+  });
+
+  let lastProcessedIndex = 0;
+  const pollTimer = setInterval(async () => {
+    try {
+      const raw = await readFile(segmentListPath, "utf8").catch(() => "");
+      if (!raw) return;
+      const entries = parseSegmentListLines(raw);
+      if (entries.length > lastProcessedIndex) {
+        lastProcessedIndex = await processCompletedSegments(entries, lastProcessedIndex, outDir, sessionId);
+        await prisma.streamSession.update({
+          where: { id: sessionId },
+          data: { totalSegments: lastProcessedIndex },
+        });
+      }
+    } catch (err) {
+      console.error(`[stream-ingest] segment list poll error for ${sessionId}:`, err);
+    }
+  }, SEGMENT_LIST_POLL_MS);
+
+  await new Promise<void>((resolve) => {
+    ffmpeg.on("exit", (code) => {
+      console.log(`[stream-ingest] ffmpeg exited (${code}) for session ${sessionId}`);
+      resolve();
+    });
+    ytdlp.on("exit", (code) => {
+      if (code !== 0) console.error(`[stream-ingest] yt-dlp exited with code ${code} for session ${sessionId}`);
+    });
+  });
+
+  clearInterval(pollTimer);
+
+  // final sweep in case segments closed between the last poll and process exit
+  const raw = await readFile(segmentListPath, "utf8").catch(() => "");
+  const entries = parseSegmentListLines(raw);
+  if (entries.length > lastProcessedIndex) {
+    lastProcessedIndex = await processCompletedSegments(entries, lastProcessedIndex, outDir, sessionId);
+  }
+
+  await prisma.streamSession.update({
+    where: { id: sessionId },
+    data: { endedAt: new Date(), totalSegments: lastProcessedIndex },
+  });
+
+  console.log(`[stream-ingest] capture finished for session ${sessionId}, ${lastProcessedIndex} segments`);
+}
+
+export function startStreamIngestWorker(): Worker<StreamIngestJob> {
+  return new Worker<StreamIngestJob>(QUEUE_NAMES.STREAM_INGEST, runCapture, {
+    connection: redisConnection,
+    concurrency: 3,
+    // Captures run for the lifetime of the stream (can be hours) — extend the
+    // default lock well beyond BullMQ's stalled-job assumptions for an MVP.
+    lockDuration: 24 * 60 * 60 * 1000,
+  });
+}
@@ -0,0 +1,10 @@
+import "dotenv/config";
+
+export const env = {
+  redisUrl: process.env.REDIS_URL ?? "redis://localhost:6379",
+  sharedMediaRoot: process.env.SHARED_MEDIA_ROOT ?? "./shared-media",
+  pollIntervalMs: Number(process.env.POLL_INTERVAL_MS ?? 60_000),
+  segmentTimeSec: Number(process.env.SEGMENT_TIME_SEC ?? 900),
+  port: Number(process.env.API_DAEMON_PORT ?? 4001),
+  forceLiveUrl: process.env.FORCE_LIVE_URL ?? "",
+};
@@ -0,0 +1,47 @@
+import { prisma } from "@streamclipper/db";
+import { env } from "./env";
+import { startPollingLoop } from "./youtubePolling";
+import { startStreamIngestWorker } from "./capture/streamIngest";
+import { streamIngestQueue } from "./queues";
+import { startServer } from "./server";
+
+async function forceLiveBypass(youtubeUrl: string) {
+  console.log(`[index] FORCE_LIVE_URL set — bypassing polling and capturing directly: ${youtubeUrl}`);
+
+  const channel = await prisma.channel.upsert({
+    where: { channelId: "forced-test-channel" },
+    update: {},
+    create: {
+      name: "Forced Test Channel",
+      youtubeHandle: "@forced-test",
+      channelId: "forced-test-channel",
+      isActive: false,
+    },
+  });
+
+  const session = await prisma.streamSession.create({
+    data: { channelId: channel.id, liveVideoId: "forced" },
+  });
+
+  await streamIngestQueue.add("start-capture", {
+    sessionId: session.id,
+    channelDbId: channel.id,
+    youtubeUrl,
+  });
+}
+
+async function main() {
+  startServer();
+  startStreamIngestWorker();
+
+  if (env.forceLiveUrl) {
+    await forceLiveBypass(env.forceLiveUrl);
+  } else {
+    startPollingLoop();
+  }
+}
+
+main().catch((err) => {
+  console.error("[index] fatal error:", err);
+  process.exit(1);
+});
@@ -0,0 +1,36 @@
+import { Queue } from "bullmq";
+import { redisConnection } from "./redis";
+
+/**
+ * Queue names are shared across the Node (api-daemon) and Python (worker)
+ * services via the protocol-compatible `bullmq` package on both sides.
+ * Keep this list in sync with apps/worker/worker/queues.py.
+ */
+export const QUEUE_NAMES = {
+  STREAM_INGEST: "stream-ingest",
+  SIGNAL_DETECTION: "signal-detection",
+  STT_SCORING: "stt-scoring",
+  // Defined for future modules (render/distribution) — no processor yet.
+  VIDEO_RENDER: "video-render",
+  POST_PUBLISH: "post-publish",
+} as const;
+
+export interface StreamIngestJob {
+  sessionId: string;
+  channelDbId: string;
+  youtubeUrl: string;
+}
+
+export interface SignalDetectionJob {
+  rawSegmentId: string;
+  filePath: string;
+  sessionId: string;
+}
+
+export const streamIngestQueue = new Queue<StreamIngestJob>(QUEUE_NAMES.STREAM_INGEST, {
+  connection: redisConnection,
+});
+
+export const signalDetectionQueue = new Queue<SignalDetectionJob>(QUEUE_NAMES.SIGNAL_DETECTION, {
+  connection: redisConnection,
+});
@@ -0,0 +1,6 @@
+import IORedis from "ioredis";
+import { env } from "./env";
+
+export const redisConnection = new IORedis(env.redisUrl, {
+  maxRetriesPerRequest: null,
+});
@@ -0,0 +1,36 @@
+import express from "express";
+import { prisma } from "@streamclipper/db";
+import { env } from "./env";
+
+export function startServer() {
+  const app = express();
+
+  app.get("/health", (_req, res) => res.json({ ok: true }));
+
+  app.get("/status", async (_req, res) => {
+    const channels = await prisma.channel.findMany({
+      include: {
+        sessions: {
+          where: { endedAt: null },
+          orderBy: { startedAt: "desc" },
+          take: 1,
+        },
+      },
+    });
+
+    res.json({
+      channels: channels.map((c) => ({
+        id: c.id,
+        name: c.name,
+        isActive: c.isActive,
+        lastCheckedAt: c.lastCheckedAt,
+        recording: c.sessions.length > 0,
+        activeSessionId: c.sessions[0]?.id ?? null,
+      })),
+    });
+  });
+
+  app.listen(env.port, () => {
+    console.log(`[server] api-daemon listening on :${env.port}`);
+  });
+}
@@ -0,0 +1,20 @@
+const BOT_TOKEN = process.env.TELEGRAM_BOT_TOKEN ?? "";
+const CHAT_ID = process.env.TELEGRAM_CHAT_ID ?? "";
+
+export async function sendTelegramMessage(text: string): Promise<void> {
+  if (!BOT_TOKEN || !CHAT_ID) {
+    console.warn("[telegram] TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID not set, skipping notification:", text);
+    return;
+  }
+
+  const url = `https://api.telegram.org/bot${BOT_TOKEN}/sendMessage`;
+  const res = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ chat_id: CHAT_ID, text, parse_mode: "Markdown" }),
+  });
+
+  if (!res.ok) {
+    console.error("[telegram] failed to send message:", res.status, await res.text());
+  }
+}
@@ -0,0 +1,80 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { prisma } from "@streamclipper/db";
+import { env } from "./env";
+import { streamIngestQueue } from "./queues";
+import { sendTelegramMessage } from "./telegram";
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Live detection goes through yt-dlp itself instead of the YouTube Data API.
+ * `search.list?eventType=live` costs 100 quota units per call against a
+ * 10,000/day free quota — polling one channel every 60s alone would need
+ * ~144,000 units/day, well over quota. yt-dlp's `/live` redirect check is
+ * free and needs no API key.
+ */
+async function findLiveVideoId(channelId: string): Promise<string | null> {
+  const liveUrl = `https://www.youtube.com/channel/${channelId}/live`;
+
+  try {
+    const { stdout } = await execFileAsync(
+      "yt-dlp",
+      ["--simulate", "--no-warnings", "--print", "%(id)s", liveUrl],
+      { timeout: 20_000 },
+    );
+    const videoId = stdout.trim().split("\n")[0];
+    return videoId || null;
+  } catch {
+    // yt-dlp exits non-zero when the channel isn't currently live
+    return null;
+  }
+}
+
+async function startSession(channel: { id: string; name: string }, liveVideoId: string) {
+  const existing = await prisma.streamSession.findFirst({
+    where: { channelId: channel.id, endedAt: null },
+  });
+  if (existing) return;
+
+  const session = await prisma.streamSession.create({
+    data: { channelId: channel.id, liveVideoId },
+  });
+
+  await streamIngestQueue.add("start-capture", {
+    sessionId: session.id,
+    channelDbId: channel.id,
+    youtubeUrl: `https://www.youtube.com/watch?v=${liveVideoId}`,
+  });
+
+  await sendTelegramMessage(`🔴 *${channel.name}* canlıya geçti, kayıt başlatılıyor.`);
+  console.log(`[youtube-polling] started session ${session.id} for channel ${channel.name}`);
+}
+
+export async function pollOnce(): Promise<void> {
+  const channels = await prisma.channel.findMany({ where: { isActive: true } });
+
+  for (const channel of channels) {
+    try {
+      const liveVideoId = await findLiveVideoId(channel.channelId);
+      await prisma.channel.update({
+        where: { id: channel.id },
+        data: { lastCheckedAt: new Date() },
+      });
+
+      if (liveVideoId) {
+        await startSession(channel, liveVideoId);
+      }
+    } catch (err) {
+      console.error(`[youtube-polling] error polling channel ${channel.name}:`, err);
+    }
+  }
+}
+
+export function startPollingLoop(): NodeJS.Timeout {
+  console.log(`[youtube-polling] polling every ${env.pollIntervalMs}ms`);
+  pollOnce().catch((err) => console.error("[youtube-polling] initial poll failed:", err));
+  return setInterval(() => {
+    pollOnce().catch((err) => console.error("[youtube-polling] poll failed:", err));
+  }, env.pollIntervalMs);
+}
@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src"]
+}