diff --git a/.env.example b/.env.example index 15033b5..94c4415 100644 --- a/.env.example +++ b/.env.example @@ -45,3 +45,8 @@ YTDLP_POT_PROVIDER_URL= # ISP proxy product specifically (e.g. Oxylabs "ISP Proxies"), not a plain # datacenter proxy — the latter has the same problem as the server itself. PROXY_URL= + +# Processed raw recordings (raw_segments) older than this get auto-deleted +# (DB row + file) — PRD's auto-purge lifecycle. Only segments already fully +# analyzed (no candidate still waiting on STT) are eligible. +RAW_SEGMENT_TTL_HOURS=24 diff --git a/apps/api-daemon/src/cleanup.ts b/apps/api-daemon/src/cleanup.ts new file mode 100644 index 0000000..6976ab4 --- /dev/null +++ b/apps/api-daemon/src/cleanup.ts @@ -0,0 +1,41 @@ +import { unlink } from "node:fs/promises"; +import { prisma } from "@streamclipper/db"; +import { env } from "./env"; + +const CHECK_INTERVAL_MS = 60 * 60 * 1000; // hourly + +/** + * PRD's auto-purge lifecycle: processed raw recordings get deleted 24h after + * capture. Only segments that are done being analyzed (PROCESSED/DISCARDED) + * and have no candidate still waiting on STT are eligible — deleting a raw + * file mid-analysis or mid-transcription would break the pipeline for it. + */ +async function cleanupOnce(): Promise { + const cutoff = new Date(Date.now() - env.rawSegmentTtlHours * 60 * 60 * 1000); + + const staleSegments = await prisma.rawSegment.findMany({ + where: { + createdAt: { lt: cutoff }, + status: { in: ["PROCESSED", "DISCARDED"] }, + candidates: { none: { status: "PENDING_STT" } }, + }, + }); + + for (const segment of staleSegments) { + await prisma.candidateSegment.deleteMany({ where: { rawSegmentId: segment.id } }); + await prisma.rawSegment.delete({ where: { id: segment.id } }); + await unlink(segment.filePath).catch(() => {}); + } + + if (staleSegments.length > 0) { + console.log(`[cleanup] removed ${staleSegments.length} raw segment(s) older than ${env.rawSegmentTtlHours}h`); + } +} + +export function startCleanupLoop(): NodeJS.Timeout { + console.log(`[cleanup] checking every ${CHECK_INTERVAL_MS}ms, TTL ${env.rawSegmentTtlHours}h`); + cleanupOnce().catch((err) => console.error("[cleanup] initial run failed:", err)); + return setInterval(() => { + cleanupOnce().catch((err) => console.error("[cleanup] run failed:", err)); + }, CHECK_INTERVAL_MS); +} diff --git a/apps/api-daemon/src/env.ts b/apps/api-daemon/src/env.ts index 4c76b45..074a3d7 100644 --- a/apps/api-daemon/src/env.ts +++ b/apps/api-daemon/src/env.ts @@ -10,4 +10,5 @@ export const env = { ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "", proxyUrl: process.env.PROXY_URL ?? "", maxConcurrentCaptures: Number(process.env.MAX_CONCURRENT_CAPTURES ?? 10), + rawSegmentTtlHours: Number(process.env.RAW_SEGMENT_TTL_HOURS ?? 24), }; diff --git a/apps/api-daemon/src/index.ts b/apps/api-daemon/src/index.ts index 1570139..40479ea 100644 --- a/apps/api-daemon/src/index.ts +++ b/apps/api-daemon/src/index.ts @@ -4,6 +4,7 @@ import { startPollingLoop } from "./youtubePolling"; import { startStreamIngestWorker } from "./capture/streamIngest"; import { streamIngestQueue } from "./queues"; import { startServer } from "./server"; +import { startCleanupLoop } from "./cleanup"; async function forceLiveBypass(youtubeUrl: string) { console.log(`[index] FORCE_LIVE_URL set — bypassing polling and capturing directly: ${youtubeUrl}`); @@ -54,6 +55,7 @@ async function main() { await closeOrphanedSessions(); startServer(); startStreamIngestWorker(); + startCleanupLoop(); if (env.forceLiveUrl) { await forceLiveBypass(env.forceLiveUrl); diff --git a/docker-compose.yml b/docker-compose.yml index 1841e1c..bbc01a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,7 @@ services: YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416 MAX_CONCURRENT_CAPTURES: ${MAX_CONCURRENT_CAPTURES:-10} PROXY_URL: ${PROXY_URL:-} + RAW_SEGMENT_TTL_HOURS: ${RAW_SEGMENT_TTL_HOURS:-24} volumes: - shared-media:/shared-media ports: