pipeline/packages/text/src/generate.ts

import {
  VideoInputSchema,
  detectInputFormat,
  LLMClient,
  getParsePrompt,
  type VideoInput,
  type TemplateType,
} from "@pipeline/shared";
import { applyLengthLimits, sanitizeVideoInput, JSON_TRUNCATION_NUDGE } from "./postprocess.js";
import type { TextModuleLlmConfig } from "./types.js";

/**
 * LLM text layer — turns raw text (from a collector or the user) into a
 * validated, sanitized VideoInput. Ports the input-normalization section of the
 * legacy parse stage verbatim:
 *  - if the text is already valid VideoInputSchema JSON, use it directly;
 *  - otherwise call the LLM with the template/source parse prompt, stripping
 *    code fences, retrying once with a conciseness nudge on JSON-parse failure
 *    (finish_reason="length" truncation is common for large trending lists);
 *  - clip over-budget fields and strip unsupported glyphs before returning.
 *
 * Reuses the proven shared prompt (getParsePrompt) unchanged — the data-source
 * metadata is merged authoritatively in the assembly layer, so the prompt does
 * not need to change for the typed-repos flow.
 */
export async function generateVideoInput(
  text: string,
  template: TemplateType,
  llm: TextModuleLlmConfig,
  source: string | undefined,
  skipLlm?: boolean
): Promise<VideoInput> {
  const detected = detectInputFormat(text);

  if (detected.format === "valid-schema") {
    return sanitizeVideoInput(detected.parsed!);
  }

  if (skipLlm || !(llm.apiKey || process.env.OPENAI_API_KEY)) {
    throw new Error(
      `Input is not valid VideoInputSchema JSON and AI processing is disabled. ` +
        `Provide structured JSON matching VideoInputSchema or enable LLM processing (set OPENAI_API_KEY or remove --skip-llm).`
    );
  }

  const client = new LLMClient(llm);
  const systemPrompt = getParsePrompt(template, source);

  const stripFences = (s: string) =>
    s.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();

  let aiParsed: unknown;
  let lastFinish: string | null = null;
  let lastRaw = "";
  for (let attempt = 0; attempt < 2 && aiParsed === undefined; attempt++) {
    const userMessage = attempt === 0 ? text : `${text}${JSON_TRUNCATION_NUDGE}`;
    const { content, finishReason } = await client.chat(systemPrompt, userMessage);
    lastFinish = finishReason;
    lastRaw = stripFences(content);
    try {
      aiParsed = JSON.parse(lastRaw);
    } catch {
      // not valid JSON yet — fall through to retry, or to the final error below
    }
  }
  if (aiParsed === undefined) {
    throw new Error(
      `AI returned invalid JSON after retry (finish_reason=${lastFinish ?? "unknown"}; ` +
        `likely truncated by the output length limit — raise max_tokens in LLMClient):\n${lastRaw.slice(0, 300)}`
    );
  }

  aiParsed = applyLengthLimits(aiParsed, template);

  const validationResult = VideoInputSchema.safeParse(aiParsed);
  if (!validationResult.success) {
    throw new Error(`AI output does not match VideoInputSchema: ${validationResult.error.message}`);
  }
  return sanitizeVideoInput(validationResult.data);
}