feat: işlenmiş ham kayıtları 24 saat sonra otomatik sil
PRD'nin auto-purge lifecycle'ı: RawSegment analiz tamamlanmış (PROCESSED/DISCARDED) ve hiçbir candidate'i PENDING_STT'de değilse (yani hâlâ transkribe edilmeyi bekleyen yoksa) 24 saatten (yapılandırı- labilir: RAW_SEGMENT_TTL_HOURS) eskiyse DB kaydı + dosyası siliniyor. Saatlik kontrol ediliyor, api-daemon başlarken de bir kere çalışıyor. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -45,3 +45,8 @@ YTDLP_POT_PROVIDER_URL=
|
|||||||
# ISP proxy product specifically (e.g. Oxylabs "ISP Proxies"), not a plain
|
# ISP proxy product specifically (e.g. Oxylabs "ISP Proxies"), not a plain
|
||||||
# datacenter proxy — the latter has the same problem as the server itself.
|
# datacenter proxy — the latter has the same problem as the server itself.
|
||||||
PROXY_URL=
|
PROXY_URL=
|
||||||
|
|
||||||
|
# Processed raw recordings (raw_segments) older than this get auto-deleted
|
||||||
|
# (DB row + file) — PRD's auto-purge lifecycle. Only segments already fully
|
||||||
|
# analyzed (no candidate still waiting on STT) are eligible.
|
||||||
|
RAW_SEGMENT_TTL_HOURS=24
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { unlink } from "node:fs/promises";
|
||||||
|
import { prisma } from "@streamclipper/db";
|
||||||
|
import { env } from "./env";
|
||||||
|
|
||||||
|
const CHECK_INTERVAL_MS = 60 * 60 * 1000; // hourly
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PRD's auto-purge lifecycle: processed raw recordings get deleted 24h after
|
||||||
|
* capture. Only segments that are done being analyzed (PROCESSED/DISCARDED)
|
||||||
|
* and have no candidate still waiting on STT are eligible — deleting a raw
|
||||||
|
* file mid-analysis or mid-transcription would break the pipeline for it.
|
||||||
|
*/
|
||||||
|
async function cleanupOnce(): Promise<void> {
|
||||||
|
const cutoff = new Date(Date.now() - env.rawSegmentTtlHours * 60 * 60 * 1000);
|
||||||
|
|
||||||
|
const staleSegments = await prisma.rawSegment.findMany({
|
||||||
|
where: {
|
||||||
|
createdAt: { lt: cutoff },
|
||||||
|
status: { in: ["PROCESSED", "DISCARDED"] },
|
||||||
|
candidates: { none: { status: "PENDING_STT" } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const segment of staleSegments) {
|
||||||
|
await prisma.candidateSegment.deleteMany({ where: { rawSegmentId: segment.id } });
|
||||||
|
await prisma.rawSegment.delete({ where: { id: segment.id } });
|
||||||
|
await unlink(segment.filePath).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (staleSegments.length > 0) {
|
||||||
|
console.log(`[cleanup] removed ${staleSegments.length} raw segment(s) older than ${env.rawSegmentTtlHours}h`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function startCleanupLoop(): NodeJS.Timeout {
|
||||||
|
console.log(`[cleanup] checking every ${CHECK_INTERVAL_MS}ms, TTL ${env.rawSegmentTtlHours}h`);
|
||||||
|
cleanupOnce().catch((err) => console.error("[cleanup] initial run failed:", err));
|
||||||
|
return setInterval(() => {
|
||||||
|
cleanupOnce().catch((err) => console.error("[cleanup] run failed:", err));
|
||||||
|
}, CHECK_INTERVAL_MS);
|
||||||
|
}
|
||||||
@@ -10,4 +10,5 @@ export const env = {
|
|||||||
ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "",
|
ytdlpPotProviderUrl: process.env.YTDLP_POT_PROVIDER_URL ?? "",
|
||||||
proxyUrl: process.env.PROXY_URL ?? "",
|
proxyUrl: process.env.PROXY_URL ?? "",
|
||||||
maxConcurrentCaptures: Number(process.env.MAX_CONCURRENT_CAPTURES ?? 10),
|
maxConcurrentCaptures: Number(process.env.MAX_CONCURRENT_CAPTURES ?? 10),
|
||||||
|
rawSegmentTtlHours: Number(process.env.RAW_SEGMENT_TTL_HOURS ?? 24),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { startPollingLoop } from "./youtubePolling";
|
|||||||
import { startStreamIngestWorker } from "./capture/streamIngest";
|
import { startStreamIngestWorker } from "./capture/streamIngest";
|
||||||
import { streamIngestQueue } from "./queues";
|
import { streamIngestQueue } from "./queues";
|
||||||
import { startServer } from "./server";
|
import { startServer } from "./server";
|
||||||
|
import { startCleanupLoop } from "./cleanup";
|
||||||
|
|
||||||
async function forceLiveBypass(youtubeUrl: string) {
|
async function forceLiveBypass(youtubeUrl: string) {
|
||||||
console.log(`[index] FORCE_LIVE_URL set — bypassing polling and capturing directly: ${youtubeUrl}`);
|
console.log(`[index] FORCE_LIVE_URL set — bypassing polling and capturing directly: ${youtubeUrl}`);
|
||||||
@@ -54,6 +55,7 @@ async function main() {
|
|||||||
await closeOrphanedSessions();
|
await closeOrphanedSessions();
|
||||||
startServer();
|
startServer();
|
||||||
startStreamIngestWorker();
|
startStreamIngestWorker();
|
||||||
|
startCleanupLoop();
|
||||||
|
|
||||||
if (env.forceLiveUrl) {
|
if (env.forceLiveUrl) {
|
||||||
await forceLiveBypass(env.forceLiveUrl);
|
await forceLiveBypass(env.forceLiveUrl);
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ services:
|
|||||||
YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416
|
YTDLP_POT_PROVIDER_URL: http://sc_pot_provider:4416
|
||||||
MAX_CONCURRENT_CAPTURES: ${MAX_CONCURRENT_CAPTURES:-10}
|
MAX_CONCURRENT_CAPTURES: ${MAX_CONCURRENT_CAPTURES:-10}
|
||||||
PROXY_URL: ${PROXY_URL:-}
|
PROXY_URL: ${PROXY_URL:-}
|
||||||
|
RAW_SEGMENT_TTL_HOURS: ${RAW_SEGMENT_TTL_HOURS:-24}
|
||||||
volumes:
|
volumes:
|
||||||
- shared-media:/shared-media
|
- shared-media:/shared-media
|
||||||
ports:
|
ports:
|
||||||
|
|||||||
Reference in New Issue
Block a user