feat: işlenmiş ham kayıtları 24 saat sonra otomatik sil
PRD'nin auto-purge lifecycle'ı: RawSegment analiz tamamlanmış (PROCESSED/DISCARDED) ve hiçbir candidate'i PENDING_STT'de değilse (yani hâlâ transkribe edilmeyi bekleyen yoksa) 24 saatten (yapılandırı- labilir: RAW_SEGMENT_TTL_HOURS) eskiyse DB kaydı + dosyası siliniyor. Saatlik kontrol ediliyor, api-daemon başlarken de bir kere çalışıyor. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -45,3 +45,8 @@ YTDLP_POT_PROVIDER_URL=
|
||||
# ISP proxy product specifically (e.g. Oxylabs "ISP Proxies"), not a plain
|
||||
# datacenter proxy — the latter has the same problem as the server itself.
|
||||
PROXY_URL=
|
||||
|
||||
# Processed raw recordings (raw_segments) older than this get auto-deleted
|
||||
# (DB row + file) — PRD's auto-purge lifecycle. Only segments already fully
|
||||
# analyzed (no candidate still waiting on STT) are eligible.
|
||||
RAW_SEGMENT_TTL_HOURS=24
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { unlink } from "node:fs/promises";
|
||||
import { prisma } from "@streamclipper/db";
|
||||
import { env } from "./env";
|
||||
|
||||
const CHECK_INTERVAL_MS = 60 * 60 * 1000; // hourly
|
||||
|
||||
/**
|
||||
* PRD's auto-purge lifecycle: processed raw recordings get deleted 24h after
|
||||
* capture. Only segments that are done being analyzed (PROCESSED/DISCARDED)
|
||||
* and have no candidate still waiting on STT are eligible — deleting a raw
|
||||
* file mid-analysis or mid-transcription would break the pipeline for it.
|
||||
*/
|
||||
async function cleanupOnce(): Promise<void> {
|
||||
const cutoff = new Date(Date.now() - env.rawSegmentTtlHours * 60 * 60 * 1000);
|
||||
|
||||
const staleSegments = await prisma.rawSegment.findMany({
|
||||
where: {
|
||||
createdAt: { lt: cutoff },
|
||||
status: { in: ["PROCESSED", "DISCARDED"] },
|
||||
candidates: { none: { status: "PENDING_STT" } },
|
||||
},
|
||||
});
|
||||
|
||||
for (const segment of staleSegments) {
|
||||
await prisma.candidateSegment.deleteMany({ where: { rawSegmentId: segment.id } });
|
||||
await prisma.rawSegment.delete({ where: { id: segment.id } });
|
||||
await unlink(segment.filePath).catch(() => {});
|
||||
}
|
||||
|
||||
if (staleSegments.length > 0) {
|
||||
console.log(`[cleanup] removed ${staleSegments.length} raw segment(s) older than ${env.rawSegmentTtlHours}h`);
|
||||
}
|
||||
}
|
||||
|
||||
export function startCleanupLoop(): NodeJS.Timeout {
|
||||
console.log(`[cleanup] checking every ${CHECK_INTERVAL_MS}ms, TTL ${env.rawSegmentTtlHours}h`);
|
||||
cleanupOnce().catch((err) => console.error("[cleanup] initial run failed:", err));
|
||||
return setInterval(() => {
|
||||
cleanupOnce().catch((err) => console.error("[cleanup] run failed:", err));
|
||||
}, CHECK_INTERVAL_MS);
|
||||
}
|
||||
@@ -10,4 +10,5 @@ export const env = {
|
||||
ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "",
|
||||
proxyUrl: process.env.PROXY_URL ?? "",
|
||||
maxConcurrentCaptures: Number(process.env.MAX_CONCURRENT_CAPTURES ?? 10),
|
||||
rawSegmentTtlHours: Number(process.env.RAW_SEGMENT_TTL_HOURS ?? 24),
|
||||
};
|
||||
|
||||
@@ -4,6 +4,7 @@ import { startPollingLoop } from "./youtubePolling";
|
||||
import { startStreamIngestWorker } from "./capture/streamIngest";
|
||||
import { streamIngestQueue } from "./queues";
|
||||
import { startServer } from "./server";
|
||||
import { startCleanupLoop } from "./cleanup";
|
||||
|
||||
async function forceLiveBypass(youtubeUrl: string) {
|
||||
console.log(`[index] FORCE_LIVE_URL set — bypassing polling and capturing directly: ${youtubeUrl}`);
|
||||
@@ -54,6 +55,7 @@ async function main() {
|
||||
await closeOrphanedSessions();
|
||||
startServer();
|
||||
startStreamIngestWorker();
|
||||
startCleanupLoop();
|
||||
|
||||
if (env.forceLiveUrl) {
|
||||
await forceLiveBypass(env.forceLiveUrl);
|
||||
|
||||
@@ -41,6 +41,7 @@ services:
|
||||
YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416
|
||||
MAX_CONCURRENT_CAPTURES: ${MAX_CONCURRENT_CAPTURES:-10}
|
||||
PROXY_URL: ${PROXY_URL:-}
|
||||
RAW_SEGMENT_TTL_HOURS: ${RAW_SEGMENT_TTL_HOURS:-24}
|
||||
volumes:
|
||||
- shared-media:/shared-media
|
||||
ports:
|
||||
|
||||
Reference in New Issue
Block a user