import { VideoInputSchema, detectInputFormat, LLMClient, getParsePrompt, type VideoInput, type TemplateType, } from "@pipeline/shared"; import { applyLengthLimits, sanitizeVideoInput, JSON_TRUNCATION_NUDGE } from "./postprocess.js"; import type { TextModuleLlmConfig } from "./types.js"; /** * LLM text layer — turns raw text (from a collector or the user) into a * validated, sanitized VideoInput. Ports the input-normalization section of the * legacy parse stage verbatim: * - if the text is already valid VideoInputSchema JSON, use it directly; * - otherwise call the LLM with the template/source parse prompt, stripping * code fences, retrying once with a conciseness nudge on JSON-parse failure * (finish_reason="length" truncation is common for large trending lists); * - clip over-budget fields and strip unsupported glyphs before returning. * * Reuses the proven shared prompt (getParsePrompt) unchanged — the data-source * metadata is merged authoritatively in the assembly layer, so the prompt does * not need to change for the typed-repos flow. */ export async function generateVideoInput( text: string, template: TemplateType, llm: TextModuleLlmConfig, source: string | undefined, skipLlm?: boolean ): Promise { const detected = detectInputFormat(text); if (detected.format === "valid-schema") { return sanitizeVideoInput(detected.parsed!); } if (skipLlm || !(llm.apiKey || process.env.OPENAI_API_KEY)) { throw new Error( `Input is not valid VideoInputSchema JSON and AI processing is disabled. ` + `Provide structured JSON matching VideoInputSchema or enable LLM processing (set OPENAI_API_KEY or remove --skip-llm).` ); } const client = new LLMClient(llm); const systemPrompt = getParsePrompt(template, source); const stripFences = (s: string) => s.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim(); let aiParsed: unknown; let lastFinish: string | null = null; let lastRaw = ""; for (let attempt = 0; attempt < 2 && aiParsed === undefined; attempt++) { const userMessage = attempt === 0 ? text : `${text}${JSON_TRUNCATION_NUDGE}`; const { content, finishReason } = await client.chat(systemPrompt, userMessage); lastFinish = finishReason; lastRaw = stripFences(content); try { aiParsed = JSON.parse(lastRaw); } catch { // not valid JSON yet — fall through to retry, or to the final error below } } if (aiParsed === undefined) { throw new Error( `AI returned invalid JSON after retry (finish_reason=${lastFinish ?? "unknown"}; ` + `likely truncated by the output length limit — raise max_tokens in LLMClient):\n${lastRaw.slice(0, 300)}` ); } aiParsed = applyLengthLimits(aiParsed, template); const validationResult = VideoInputSchema.safeParse(aiParsed); if (!validationResult.success) { throw new Error(`AI output does not match VideoInputSchema: ${validationResult.error.message}`); } return sanitizeVideoInput(validationResult.data); }