3
0

3 Revīzijas 073d1f39d6 ... 30adfed2d6

Autors SHA1 Ziņojums Datums
  lkatzey 30adfed2d6 支持一次性生成多平台内容 1 mēnesi atpakaļ
  lkatzey e6eb3062cc 取消字幕高亮效果 1 mēnesi atpakaļ
  lkatzey 1a37223e59 增加数据采集模块 1 mēnesi atpakaļ
36 mainītis faili ar 1095 papildinājumiem un 189 dzēšanām
  1. 10 9
      .env.example
  2. 1 0
      apps/cli/package.json
  3. 67 0
      apps/cli/src/commands/collect.ts
  4. 93 37
      apps/cli/src/commands/render.ts
  5. 2 0
      apps/cli/src/index.ts
  6. 1 0
      apps/cli/tsconfig.json
  7. 2 0
      apps/web/package.json
  8. 34 0
      apps/web/src/app/api/collect/route.ts
  9. 16 0
      apps/web/src/app/api/collect/sources/route.ts
  10. 39 26
      apps/web/src/app/api/render/route.ts
  11. 192 19
      apps/web/src/app/create/page.tsx
  12. 3 2
      apps/web/src/app/jobs/[id]/page.tsx
  13. 2 2
      apps/web/src/app/jobs/page.tsx
  14. 36 0
      apps/web/src/app/settings/page.tsx
  15. 22 0
      apps/web/src/lib/config.ts
  16. 2 1
      apps/web/src/lib/job-store.ts
  17. 33 14
      config/default.yaml
  18. 24 0
      packages/collect/package.json
  19. 96 0
      packages/collect/src/collectors/github-daily.ts
  20. 55 0
      packages/collect/src/collectors/github-repo.ts
  21. 42 0
      packages/collect/src/collectors/github-trending.ts
  22. 7 0
      packages/collect/src/index.ts
  23. 27 0
      packages/collect/src/registry.ts
  24. 26 0
      packages/collect/src/types.ts
  25. 12 0
      packages/collect/tsconfig.json
  26. 43 31
      packages/core/src/pipeline.ts
  27. 63 8
      packages/core/src/stages/parse.ts
  28. 15 0
      packages/core/src/stages/tts.ts
  29. 2 2
      packages/shared/src/index.ts
  30. 25 0
      packages/shared/src/llm/detect-input.ts
  31. 2 0
      packages/shared/src/llm/index.ts
  32. 3 3
      packages/shared/src/types/pipeline.ts
  33. 5 34
      packages/templates/src/base/components/subtitle-bar.tsx
  34. 1 1
      packages/tts/src/providers/openai-tts.ts
  35. 19 0
      pnpm-lock.yaml
  36. 73 0
      test/fixtures/test-news.md

+ 10 - 9
.env.example

@@ -1,20 +1,21 @@
-# Environment variables (copy to .env and fill in values)
+# Environment variables — secrets and endpoint URLs only.
+# Copy this file to .env and fill in values.
+# Model, voice, and other business settings go in config/default.yaml or CLI flags.
 
-# LLM
+# --- LLM ---
 OPENAI_API_KEY=
-OPENAI_BASE_URL=https://api.openai.com/v1
+OPENAI_BASE_URL=
 
-# TTS — OpenAI TTS (default)
+# --- TTS: OpenAI-compatible ---
 OPENAI_TTS_API_KEY=
-OPENAI_TTS_BASE_URL=https://new-api.corp.shuidi.tech/v1
-OPENAI_TTS_MODEL=seed-tts-1.1
+OPENAI_TTS_BASE_URL=
 
-# TTS — Fish Audio
+# --- TTS: Fish Audio ---
 FISH_AUDIO_API_KEY=
 
-# TTS — MiniMax
+# --- TTS: MiniMax ---
 MINIMAX_API_KEY=
 MINIMAX_GROUP_ID=
 
-# TTS — ElevenLabs
+# --- TTS: ElevenLabs ---
 ELEVENLABS_API_KEY=

+ 1 - 0
apps/cli/package.json

@@ -13,6 +13,7 @@
   "dependencies": {
     "@pipeline/core": "workspace:*",
     "@pipeline/shared": "workspace:*",
+    "@pipeline/collect": "workspace:*",
     "@pipeline/tts": "workspace:*",
     "@pipeline/templates": "workspace:*",
     "commander": "^13.0.0",

+ 67 - 0
apps/cli/src/commands/collect.ts

@@ -0,0 +1,67 @@
+import { Command } from "commander";
+import { getCollector, listCollectorNames } from "@pipeline/collect";
+import { writeFileSync, existsSync, readFileSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { parse as parseYaml } from "yaml";
+
+export const collectCommand = new Command("collect")
+  .description("Collect data from external sources")
+  .requiredOption("-s, --source <name>", "Data source name")
+  .option("-o, --output <path>", "Output file path (prints to stdout if omitted)")
+  .option("--config <path>", "Config file path")
+  .option("--owner <owner>", "Repository owner (for github-repo)")
+  .option("--repo <repo>", "Repository name (for github-repo)")
+  .action(async (opts) => {
+    const config = loadConfig(opts.config);
+    const collectorConfig = config?.collect?.[opts.source];
+    const collector = getCollector(opts.source, collectorConfig);
+    const args: Record<string, string> = {};
+    if (opts.owner) args.owner = opts.owner;
+    if (opts.repo) args.repo = opts.repo;
+
+    console.error(`Collecting from ${opts.source}...`);
+
+    try {
+      const result = await collector.collect(
+        Object.keys(args).length > 0 ? { args } : undefined
+      );
+
+      const content =
+        result.type === "json"
+          ? JSON.stringify(result.content, null, 2)
+          : result.content;
+
+      if (opts.output) {
+        const outputPath = resolve(opts.output);
+        writeFileSync(outputPath, content, "utf-8");
+        console.error(`Written to ${outputPath}`);
+      } else {
+        process.stdout.write(content);
+      }
+    } catch (err: any) {
+      console.error(`Error: ${err.message}`);
+      process.exit(1);
+    }
+  });
+
+function loadConfig(configPath?: string): Record<string, any> | null {
+  const cliDir = dirname(new URL(import.meta.url).pathname);
+  const projectRoot = resolve(cliDir, "../../../../");
+
+  const paths = [
+    configPath,
+    "pipeline.config.yaml",
+    "pipeline.config.yml",
+    resolve(projectRoot, "config/default.yaml"),
+    "config/default.yaml",
+  ].filter(Boolean) as string[];
+
+  for (const p of paths) {
+    const resolved = resolve(p);
+    if (existsSync(resolved)) {
+      const content = readFileSync(resolved, "utf-8");
+      return parseYaml(content);
+    }
+  }
+  return null;
+}

+ 93 - 37
apps/cli/src/commands/render.ts

@@ -1,6 +1,7 @@
 import { Command } from "commander";
 import { runPipeline, type PipelineConfig } from "@pipeline/core";
 import { PLATFORM_PRESETS, TEMPLATE_TYPES, PLATFORM_PRESET_KEYS } from "@pipeline/shared";
+import { getCollector, listCollectorNames } from "@pipeline/collect";
 import { readFileSync, existsSync } from "node:fs";
 import { resolve, dirname } from "node:path";
 import { parse as parseYaml } from "yaml";
@@ -9,20 +10,39 @@ import { parse as parseYaml } from "yaml";
 const skipTts = (opts: any) => opts.tts === false;
 
 export const renderCommand = new Command("render")
-  .description("Generate video from structured JSON input")
-  .argument("<input>", "Input JSON file path")
+  .description("Generate video from text, markdown, or structured JSON input")
+  .argument("[input]", "Input file path (JSON, markdown, or plain text). Required unless --source is used.")
   .requiredOption("-t, --template <type>", `Template: ${TEMPLATE_TYPES.join("|")}`)
   .option("-p, --platform <preset>", "Platform preset", "bilibili")
-  .option("--tts-provider <provider>", "TTS provider")
-  .option("--voice <id>", "Voice ID override")
   .option("-o, --output <dir>", "Output directory", "./output")
   .option("--assets-dir <dir>", "Image assets directory (defaults to JSON file location)")
   .option("--config <path>", "Config file path")
+  .option("-v, --verbose", "Verbose logging")
+
+  // Data source
+  .option("--source <name>", "Collect data from source before rendering")
+  .option("--source-owner <owner>", "Repo owner (for github-repo source)")
+  .option("--source-repo <repo>", "Repo name (for github-repo source)")
+
+  // LLM
+  .option("--skip-llm", "Skip LLM processing (require valid VideoInputSchema JSON)")
+  .option("--llm-model <model>", "LLM model name")
+  .option("--llm-base-url <url>", "LLM API base URL")
+  .option("--llm-api-key <key>", "LLM API key")
+
+  // TTS — general
+  .option("--tts-provider <provider>", "TTS provider (openai-tts|fish-audio|minimax|elevenlabs)")
+  .option("--voice <id>", "Voice ID")
+  .option("--tts-model <model>", "TTS model name (provider-specific)")
+  .option("--tts-speed <speed>", "TTS speech speed (0.5-2.0)", parseFloat)
+  .option("--tts-format <format>", "TTS audio format (mp3|wav|pcm)")
   .option("--no-tts", "Skip TTS (generate silent video)")
+
+  // Alignment
   .option("--alignment <mode>", "Timestamp alignment: whisper|native")
   .option("--whisper-model <model>", "Whisper model size (tiny/base/small/medium/large)", "base")
   .option("--alignment-language <lang>", "Language hint for whisper (e.g. zh, en)")
-  .option("-v, --verbose", "Verbose logging")
+
   .action(async (input, opts) => {
     const template = opts.template as string;
     if (!TEMPLATE_TYPES.includes(template as any)) {
@@ -30,35 +50,59 @@ export const renderCommand = new Command("render")
       process.exit(1);
     }
 
-    const platform = opts.platform as string;
-    if (!PLATFORM_PRESET_KEYS.includes(platform as any)) {
-      console.error(`Error: Invalid platform "${platform}". Use: ${PLATFORM_PRESET_KEYS.join(", ")}`);
+    const platformStr = opts.platform as string;
+    const platforms = platformStr.split(",").map((p: string) => p.trim());
+    const invalidPlatforms = platforms.filter((p: string) => !PLATFORM_PRESET_KEYS.includes(p as any));
+    if (invalidPlatforms.length > 0) {
+      console.error(`Error: Invalid platform(s): ${invalidPlatforms.join(", ")}. Use: ${PLATFORM_PRESET_KEYS.join(", ")}`);
       process.exit(1);
     }
 
-    // Read JSON input file
-    const inputPath = resolve(input);
-    if (!existsSync(inputPath)) {
-      console.error(`Error: Input file not found: ${inputPath}`);
-      process.exit(1);
-    }
-    const text = readFileSync(inputPath, "utf-8");
+    const config = loadConfig(opts.config);
 
-    // Validate it's valid JSON
-    try {
-      JSON.parse(text);
-    } catch {
-      console.error("Error: Input file is not valid JSON");
-      process.exit(1);
+    // Collect data from source or read from file
+    let text: string;
+    let inputDir: string;
+
+    if (opts.source) {
+      const available = listCollectorNames();
+      if (!available.includes(opts.source)) {
+        console.error(`Error: Unknown source "${opts.source}". Available: ${available.join(", ")}`);
+        process.exit(1);
+      }
+      const collectorConfig = config?.collect?.[opts.source];
+      const collector = getCollector(opts.source, collectorConfig);
+      const args: Record<string, string> = {};
+      if (opts.sourceOwner) args.owner = opts.sourceOwner;
+      if (opts.sourceRepo) args.repo = opts.sourceRepo;
+
+      console.log(`Collecting from ${opts.source}...`);
+      const result = await collector.collect(
+        Object.keys(args).length > 0 ? { args } : undefined
+      );
+      text = result.type === "json" ? JSON.stringify(result.content) : result.content;
+      inputDir = resolve(process.cwd());
+    } else {
+      if (!input) {
+        console.error("Error: <input> file path is required when --source is not provided");
+        process.exit(1);
+      }
+      const inputPath = resolve(input);
+      if (!existsSync(inputPath)) {
+        console.error(`Error: Input file not found: ${inputPath}`);
+        process.exit(1);
+      }
+      text = readFileSync(inputPath, "utf-8");
+      inputDir = opts.assetsDir ? resolve(opts.assetsDir) : dirname(inputPath);
     }
 
-    const config = loadConfig(opts.config);
+    const providerName = opts.ttsProvider || config?.tts?.provider || "openai-tts";
+    const providerConfig = config?.tts?.[providerName];
 
     const cliDir = dirname(new URL(import.meta.url).pathname);
     const projectRoot = resolve(cliDir, "../../../../");
     const templatesEntry = resolve(projectRoot, "packages/templates/src/entry.ts");
     const assetsRoot = resolve(projectRoot, "assets");
-    const inputDir = opts.assetsDir ? resolve(opts.assetsDir) : dirname(inputPath);
 
     const noTts = skipTts(opts);
     const flags: string[] = [];
@@ -66,15 +110,16 @@ export const renderCommand = new Command("render")
 
     const pipelineConfig: PipelineConfig = {
       llm: {
-        baseURL: config?.llm?.baseURL,
-        apiKey: config?.llm?.apiKey,
-        model: config?.llm?.model || "gpt-4o",
+        baseURL: opts.llmBaseUrl || config?.llm?.baseURL,
+        apiKey: opts.llmApiKey || config?.llm?.apiKey,
+        model: opts.llmModel || config?.llm?.model || "gpt-4o",
       },
       tts: {
-        provider: opts.ttsProvider || config?.tts?.provider || "openai-tts",
-        voiceId: opts.voice || config?.tts?.[opts.ttsProvider || config?.tts?.provider]?.defaultVoice,
-        format: "mp3",
-        speed: 1.0,
+        provider: providerName,
+        voiceId: opts.voice || providerConfig?.defaultVoice,
+        model: opts.ttsModel || providerConfig?.model,
+        format: opts.ttsFormat || config?.tts?.format || "mp3",
+        speed: opts.ttsSpeed || config?.tts?.speed || 1.0,
       },
       output: {
         dir: resolve(opts.output || config?.output?.dir || "./output"),
@@ -82,38 +127,45 @@ export const renderCommand = new Command("render")
       assets: { root: assetsRoot, inputDir },
       templates: { entryPoint: templatesEntry },
       skipTts: noTts,
+      skipLlm: opts.skipLlm,
       alignment: buildAlignmentConfig(opts, config),
     };
 
     const stages = ["parse", "tts", "assets", "compose", "render", "export"];
 
-    console.log(`\nPipeline: generating ${template} video for ${platform}`);
+    console.log(`\nPipeline: generating ${template} video for ${platforms.join(", ")}`);
     console.log(`Input: ${text.length} chars${flags.length ? " | Flags: " + flags.join(", ") : ""}\n`);
 
     const job = await runPipeline(
       {
         text,
         template: template as any,
-        platform: platform as any,
-        ttsProvider: (opts.ttsProvider || config?.tts?.provider || "openai-tts") as any,
-        voiceId: opts.voice,
+        platforms: platforms as any,
+        ttsProvider: providerName as any,
+        voiceId: pipelineConfig.tts.voiceId,
         outputPath: resolve(opts.output || "./output"),
       },
       pipelineConfig,
       {
         onStageStart: (stage) => {
+          const [name, plat] = stage.split(":");
+          const platformLabel = plat ? ` [${plat}]` : "";
           const labels: Record<string, string> = {
-            parse: "Parsing JSON input",
+            parse: opts.skipLlm ? "Parsing JSON input" : "Parsing input (AI-assisted if needed)",
             tts: noTts ? "Generating silent audio" : `Generating narration${pipelineConfig.alignment?.provider === "whisper" ? " (whisper alignment)" : ""}`,
             assets: "Resolving assets",
             compose: "Composing scenes",
             render: "Rendering video",
             export: "Exporting",
           };
-          console.log(`  [${stages.indexOf(stage) + 1}/6] ${labels[stage] || stage}...`);
+          const stepNum = stages.indexOf(name as string) + 1;
+          console.log(`  [${stepNum}/6] ${labels[name] || name}${platformLabel}...`);
         },
         onStageComplete: (stage) => {
-          console.log(`  [${stages.indexOf(stage) + 1}/6] ${stage} done`);
+          const [name, plat] = stage.split(":");
+          const platformLabel = plat ? ` [${plat}]` : "";
+          const stepNum = stages.indexOf(name as string) + 1;
+          console.log(`  [${stepNum}/6] ${name}${platformLabel} done`);
         },
         onError: (stage, error) => {
           console.error(`\n  Error in ${stage}: ${error.message}`);
@@ -150,10 +202,14 @@ function buildAlignmentConfig(
 }
 
 function loadConfig(configPath?: string): Record<string, any> | null {
+  const cliDir = dirname(new URL(import.meta.url).pathname);
+  const projectRoot = resolve(cliDir, "../../../../");
+
   const paths = [
     configPath,
     "pipeline.config.yaml",
     "pipeline.config.yml",
+    resolve(projectRoot, "config/default.yaml"),
     "config/default.yaml",
   ].filter(Boolean) as string[];
 

+ 2 - 0
apps/cli/src/index.ts

@@ -7,6 +7,7 @@ import { previewCommand } from "./commands/preview.js";
 import { templatesCommand } from "./commands/templates.js";
 import { voicesCommand } from "./commands/voices.js";
 import { initCommand } from "./commands/init.js";
+import { collectCommand } from "./commands/collect.js";
 
 // Auto-load .env from monorepo root (3 levels up from dist/index.js)
 const envPath = resolve(import.meta.dirname, "../../../.env");
@@ -33,6 +34,7 @@ program
   .version("0.0.1");
 
 program.addCommand(renderCommand);
+program.addCommand(collectCommand);
 program.addCommand(previewCommand);
 program.addCommand(templatesCommand);
 program.addCommand(voicesCommand);

+ 1 - 0
apps/cli/tsconfig.json

@@ -8,6 +8,7 @@
   "include": ["src"],
   "references": [
     { "path": "../../packages/shared" },
+    { "path": "../../packages/collect" },
     { "path": "../../packages/core" },
     { "path": "../../packages/tts" },
     { "path": "../../packages/templates" }

+ 2 - 0
apps/web/package.json

@@ -9,9 +9,11 @@
   },
   "dependencies": {
     "@pipeline/shared": "workspace:*",
+    "@pipeline/collect": "workspace:*",
     "next": "^15.3.0",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
+    "yaml": "^2.7.0",
     "zod": "^3.24.0"
   },
   "devDependencies": {

+ 34 - 0
apps/web/src/app/api/collect/route.ts

@@ -0,0 +1,34 @@
+import { NextRequest, NextResponse } from "next/server";
+import { getCollector, listCollectorNames } from "@pipeline/collect";
+import { loadConfig } from "@/lib/config";
+
+export async function POST(request: NextRequest) {
+  const body = await request.json();
+  const { source, args } = body as { source: string; args?: Record<string, string> };
+
+  if (!source) {
+    return NextResponse.json({ error: "source is required" }, { status: 400 });
+  }
+
+  const available = listCollectorNames();
+  if (!available.includes(source)) {
+    return NextResponse.json(
+      { error: `Unknown source: ${source}. Available: ${available.join(", ")}` },
+      { status: 400 }
+    );
+  }
+
+  const config = loadConfig();
+  const collectorConfig = config?.collect?.[source];
+  const collector = getCollector(source, collectorConfig);
+
+  try {
+    const result = await collector.collect(args ? { args } : undefined);
+    const content =
+      result.type === "json" ? JSON.stringify(result.content, null, 2) : result.content;
+
+    return NextResponse.json({ content, type: result.type });
+  } catch (err: any) {
+    return NextResponse.json({ error: err.message }, { status: 500 });
+  }
+}

+ 16 - 0
apps/web/src/app/api/collect/sources/route.ts

@@ -0,0 +1,16 @@
+import { NextResponse } from "next/server";
+import { listCollectorNames } from "@pipeline/collect";
+import { loadConfig } from "@/lib/config";
+
+export async function GET() {
+  const config = loadConfig();
+  const names = listCollectorNames();
+  const sources = names.map((name) => {
+    const sourceConfig = config?.collect?.[name] ?? {};
+    return {
+      name,
+      url: sourceConfig.url ?? "",
+    };
+  });
+  return NextResponse.json({ sources });
+}

+ 39 - 26
apps/web/src/app/api/render/route.ts

@@ -9,9 +9,12 @@ export async function POST(request: Request) {
   const body = await request.json();
   const { input, options } = body as {
     input: {
-      text: string;
+      text?: string;
+      source?: string;
+      sourceArgs?: Record<string, string>;
       template: string;
-      platform: string;
+      platform?: string;
+      platforms?: string[];
       ttsProvider: string;
       voiceId?: string;
     };
@@ -20,21 +23,35 @@ export async function POST(request: Request) {
     };
   };
 
-  const job = createJob(input);
+  const platforms: string[] = input.platforms
+    ?? (input.platform ? input.platform.split(",").map(p => p.trim()) : ["bilibili"]);
+
+  if (!input.text && !input.source) {
+    return NextResponse.json({ error: "text or source is required" }, { status: 400 });
+  }
+
+  const job = createJob({ ...input, text: input.text ?? "", platforms });
 
   const jobDir = join(tmpdir(), "pipeline-jobs", job.id);
   mkdirSync(jobDir, { recursive: true });
-  const inputFile = join(jobDir, "input.json");
-  writeFileSync(inputFile, input.text, "utf-8");
 
   const cliArgs = [
     "render",
-    inputFile,
     "-t", input.template,
-    "-p", input.platform,
+    "-p", platforms.join(","),
     "--tts-provider", input.ttsProvider,
     "--output", resolve("output"),
   ];
+
+  if (input.source) {
+    cliArgs.push("--source", input.source);
+    if (input.sourceArgs?.owner) cliArgs.push("--source-owner", input.sourceArgs.owner);
+    if (input.sourceArgs?.repo) cliArgs.push("--source-repo", input.sourceArgs.repo);
+  } else {
+    const inputFile = join(jobDir, "input.json");
+    writeFileSync(inputFile, input.text!, "utf-8");
+    cliArgs.splice(1, 0, inputFile);
+  }
   if (input.voiceId) {
     cliArgs.push("--voice", input.voiceId);
   }
@@ -66,29 +83,25 @@ export async function POST(request: Request) {
 
   child.on("close", (code: number) => {
     if (code === 0) {
-      const match = stdout.match(/-> (.+\.mp4)/);
-      const filePath = match ? match[1].trim() : "";
+      const mp4Matches = [...stdout.matchAll(/-> (.+\.mp4)/g)];
+      const filePaths = mp4Matches.map(m => m[1].trim());
 
-      let fileSizeBytes = 0;
-      if (filePath) {
-        try {
-          fileSizeBytes = statSync(filePath).size;
-        } catch {}
-      }
+      const outputFiles = filePaths.map(filePath => {
+        let fileSizeBytes = 0;
+        try { fileSizeBytes = statSync(filePath).size; } catch {}
+        const isPortrait = filePath.includes("douyin") || filePath.includes("9x16");
+        return {
+          filePath,
+          fileSizeBytes,
+          width: isPortrait ? 1080 : 1920,
+          height: isPortrait ? 1920 : 1080,
+          durationSeconds: 0,
+        };
+      });
 
       updateJob(job.id, {
         status: "completed",
-        outputFiles: filePath
-          ? [
-              {
-                filePath,
-                fileSizeBytes,
-                width: input.platform.includes("9:16") || input.platform.includes("douyin") ? 1080 : 1920,
-                height: input.platform.includes("9:16") || input.platform.includes("douyin") ? 1920 : 1080,
-                durationSeconds: 0,
-              },
-            ]
-          : undefined,
+        outputFiles: outputFiles.length > 0 ? outputFiles : undefined,
       });
     } else {
       updateJob(job.id, {

+ 192 - 19
apps/web/src/app/create/page.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useState } from "react";
+import { useState, useEffect } from "react";
 import { TEMPLATE_TYPES, PLATFORM_PRESET_KEYS } from "@pipeline/shared";
 
 const TEMPLATES_INFO: Record<string, { label: string; desc: string; color: string }> = {
@@ -42,9 +42,17 @@ const JSON_PLACEHOLDER = `{
 }`;
 
 type Step = "input" | "style" | "voice" | "review";
+type InputMode = "manual" | "source";
+
+interface SourceInfo {
+  name: string;
+  url: string;
+  config: Record<string, any>;
+}
 
 export default function CreatePage() {
   const [step, setStep] = useState<Step>("input");
+  const [inputMode, setInputMode] = useState<InputMode>("manual");
   const [text, setText] = useState("");
   const [template, setTemplate] = useState("news");
   const [platform, setPlatform] = useState("bilibili");
@@ -54,8 +62,27 @@ export default function CreatePage() {
   const [jobId, setJobId] = useState<string | null>(null);
   const [error, setError] = useState<string | null>(null);
 
+  // Data source state
+  const [sources, setSources] = useState<SourceInfo[]>([]);
+  const [selectedSource, setSelectedSource] = useState("");
+  const [sourceArgs, setSourceArgs] = useState<Record<string, string>>({});
+  const [collecting, setCollecting] = useState(false);
+  const [collectError, setCollectError] = useState<string | null>(null);
+
+  useEffect(() => {
+    fetch("/api/collect/sources")
+      .then((res) => res.json())
+      .then((data) => {
+        setSources(data.sources ?? []);
+        if (data.sources?.length > 0 && !selectedSource) {
+          setSelectedSource(data.sources[0].name);
+        }
+      })
+      .catch(() => {});
+  }, []);
+
   const steps: { key: Step; label: string }[] = [
-    { key: "input", label: "1. JSON Input" },
+    { key: "input", label: "1. Input" },
     { key: "style", label: "2. Template" },
     { key: "voice", label: "3. Voice" },
     { key: "review", label: "4. Review" },
@@ -63,16 +90,49 @@ export default function CreatePage() {
 
   const stepIndex = steps.findIndex((s) => s.key === step);
 
+  async function handleCollect() {
+    if (!selectedSource) return;
+    setCollecting(true);
+    setCollectError(null);
+    try {
+      const res = await fetch("/api/collect", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ source: selectedSource, args: sourceArgs }),
+      });
+      const data = await res.json();
+      if (data.error) {
+        setCollectError(data.error);
+      } else {
+        setText(data.content);
+      }
+    } catch (err: any) {
+      setCollectError(err.message);
+    } finally {
+      setCollecting(false);
+    }
+  }
+
   async function handleRender() {
     setSubmitting(true);
     setError(null);
     try {
+      const payload: any = {
+        input: { template, platform, ttsProvider, voiceId },
+      };
+      if (inputMode === "source" && selectedSource) {
+        payload.input.source = selectedSource;
+        if (Object.keys(sourceArgs).length > 0) {
+          payload.input.sourceArgs = sourceArgs;
+        }
+      } else {
+        payload.input.text = text;
+      }
+
       const res = await fetch("/api/render", {
         method: "POST",
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          input: { text, template, platform, ttsProvider, voiceId },
-        }),
+        body: JSON.stringify(payload),
       });
       const data = await res.json();
       if (data.jobId) {
@@ -87,6 +147,13 @@ export default function CreatePage() {
     }
   }
 
+  const canProceed =
+    inputMode === "manual" ? text.trim().length > 0 : selectedSource.length > 0;
+
+  // Source-specific arg fields
+  const currentSource = sources.find((s) => s.name === selectedSource);
+  const needsOwnerRepo = currentSource?.name === "github-repo";
+
   if (jobId) {
     return (
       <div>
@@ -109,7 +176,7 @@ export default function CreatePage() {
     <div>
       <div className="page-header">
         <h1>Create Video</h1>
-        <p>Provide structured JSON to generate a video</p>
+        <p>Provide content manually or collect from external data sources</p>
       </div>
 
       <div className="steps">
@@ -126,19 +193,119 @@ export default function CreatePage() {
       <div className="card" style={{ padding: 32 }}>
         {step === "input" && (
           <div>
-            <div className="form-group">
-              <label>Video JSON</label>
-              <textarea
-                className="form-textarea"
-                value={text}
-                onChange={(e) => setText(e.target.value)}
-                placeholder={JSON_PLACEHOLDER}
-                style={{ minHeight: 300, fontFamily: "monospace", fontSize: 13 }}
-              />
+            {/* Mode toggle */}
+            <div style={{ display: "flex", gap: 12, marginBottom: 20 }}>
+              <button
+                className={`btn ${inputMode === "manual" ? "btn-primary" : "btn-secondary"}`}
+                onClick={() => setInputMode("manual")}
+              >
+                Manual Input
+              </button>
+              <button
+                className={`btn ${inputMode === "source" ? "btn-primary" : "btn-secondary"}`}
+                onClick={() => setInputMode("source")}
+              >
+                Data Source
+              </button>
             </div>
+
+            {inputMode === "manual" && (
+              <div className="form-group">
+                <label>Video JSON</label>
+                <textarea
+                  className="form-textarea"
+                  value={text}
+                  onChange={(e) => setText(e.target.value)}
+                  placeholder={JSON_PLACEHOLDER}
+                  style={{ minHeight: 300, fontFamily: "monospace", fontSize: 13 }}
+                />
+              </div>
+            )}
+
+            {inputMode === "source" && (
+              <div>
+                <div className="form-group">
+                  <label>Data Source</label>
+                  <select
+                    className="form-select"
+                    value={selectedSource}
+                    onChange={(e) => {
+                      setSelectedSource(e.target.value);
+                      setSourceArgs({});
+                      setCollectError(null);
+                    }}
+                  >
+                    {sources.map((s) => (
+                      <option key={s.name} value={s.name}>{s.name}</option>
+                    ))}
+                  </select>
+                </div>
+
+                {needsOwnerRepo && (
+                  <div style={{ display: "grid", gridTemplateColumns: "1fr 1fr", gap: 12 }}>
+                    <div className="form-group">
+                      <label>Owner</label>
+                      <input
+                        className="form-input"
+                        value={sourceArgs.owner ?? ""}
+                        onChange={(e) =>
+                          setSourceArgs((prev) => ({ ...prev, owner: e.target.value }))
+                        }
+                        placeholder="e.g. facebook"
+                      />
+                    </div>
+                    <div className="form-group">
+                      <label>Repo</label>
+                      <input
+                        className="form-input"
+                        value={sourceArgs.repo ?? ""}
+                        onChange={(e) =>
+                          setSourceArgs((prev) => ({ ...prev, repo: e.target.value }))
+                        }
+                        placeholder="e.g. react"
+                      />
+                    </div>
+                  </div>
+                )}
+
+                {currentSource && (
+                  <div style={{ fontSize: 13, color: "var(--text-muted)", marginBottom: 16 }}>
+                    URL: <code>{currentSource.url}</code>
+                  </div>
+                )}
+
+                <button
+                  className="btn btn-secondary"
+                  disabled={collecting || !selectedSource}
+                  onClick={handleCollect}
+                  style={{ marginBottom: 16 }}
+                >
+                  {collecting ? "Collecting..." : "Collect Data"}
+                </button>
+
+                {collectError && (
+                  <div style={{ color: "var(--error)", marginBottom: 16, fontSize: 14 }}>
+                    {collectError}
+                  </div>
+                )}
+
+                {text && (
+                  <div className="form-group">
+                    <label>Collected Content (editable)</label>
+                    <textarea
+                      className="form-textarea"
+                      value={text}
+                      onChange={(e) => setText(e.target.value)}
+                      style={{ minHeight: 200, fontFamily: "monospace", fontSize: 13 }}
+                    />
+                  </div>
+                )}
+              </div>
+            )}
+
             <button
               className="btn btn-primary"
-              disabled={!text.trim()}
+              disabled={!canProceed}
               onClick={() => setStep("style")}
             >
               Next
@@ -245,12 +412,18 @@ export default function CreatePage() {
                 <div style={{ fontWeight: 600 }}>{ttsProvider}</div>
               </div>
               <div className="card">
-                <div style={{ fontSize: 12, color: "var(--text-muted)", marginBottom: 4 }}>Input Size</div>
-                <div style={{ fontWeight: 600 }}>{text.length} chars</div>
+                <div style={{ fontSize: 12, color: "var(--text-muted)", marginBottom: 4 }}>Input</div>
+                <div style={{ fontWeight: 600 }}>
+                  {inputMode === "source"
+                    ? `Source: ${selectedSource}`
+                    : `${text.length} chars`}
+                </div>
               </div>
             </div>
             <div className="card" style={{ marginBottom: 24 }}>
-              <div style={{ fontSize: 12, color: "var(--text-muted)", marginBottom: 8 }}>JSON Preview</div>
+              <div style={{ fontSize: 12, color: "var(--text-muted)", marginBottom: 8 }}>
+                {inputMode === "source" ? "Source Content" : "JSON Preview"}
+              </div>
               <pre style={{ fontSize: 13, color: "var(--text-muted)", maxHeight: 120, overflow: "hidden" }}>
                 {text.slice(0, 500)}{text.length > 500 ? "\n..." : ""}
               </pre>

+ 3 - 2
apps/web/src/app/jobs/[id]/page.tsx

@@ -17,7 +17,8 @@ interface Job {
   input: {
     text: string;
     template: string;
-    platform: string;
+    platform?: string;
+    platforms?: string[];
     ttsProvider: string;
   };
   createdAt: number;
@@ -112,7 +113,7 @@ export default function JobDetailPage({ params }: { params: Promise<{ id: string
         </div>
         <div className="card">
           <div style={{ fontSize: 12, color: "var(--text-muted)", marginBottom: 4 }}>Platform</div>
-          <div style={{ fontWeight: 600 }}>{job.input.platform}</div>
+          <div style={{ fontWeight: 600 }}>{job.input.platforms?.join(", ") ?? job.input.platform}</div>
         </div>
         <div className="card">
           <div style={{ fontSize: 12, color: "var(--text-muted)", marginBottom: 4 }}>TTS Provider</div>

+ 2 - 2
apps/web/src/app/jobs/page.tsx

@@ -6,7 +6,7 @@ import Link from "next/link";
 interface Job {
   id: string;
   status: "pending" | "running" | "completed" | "failed";
-  input: { template: string; platform: string };
+  input: { template: string; platform?: string; platforms?: string[] };
   createdAt: number;
   updatedAt: number;
   error?: string;
@@ -58,7 +58,7 @@ export default function JobsPage() {
               <div className="card card-hover" style={{ display: "flex", alignItems: "center", justifyContent: "space-between" }}>
                 <div>
                   <div style={{ fontWeight: 600, marginBottom: 4 }}>
-                    {job.input.template} / {job.input.platform}
+                    {job.input.template} / {(job.input.platforms ?? [job.input.platform]).join(", ")}
                   </div>
                   <div style={{ fontSize: 13, color: "var(--text-muted)" }}>
                     {new Date(job.createdAt).toLocaleString("zh-CN")}

+ 36 - 0
apps/web/src/app/settings/page.tsx

@@ -31,6 +31,7 @@ export default function SettingsPage() {
   const [loaded, setLoaded] = useState(false);
   const [saving, setSaving] = useState(false);
   const [saved, setSaved] = useState(false);
+  const [sources, setSources] = useState<{ name: string; url: string }[]>([]);
 
   useEffect(() => {
     fetch("/api/settings")
@@ -40,6 +41,11 @@ export default function SettingsPage() {
         setLoaded(true);
       })
       .catch(() => setLoaded(true));
+
+    fetch("/api/collect/sources")
+      .then((r) => r.json())
+      .then((data) => setSources(data.sources ?? []))
+      .catch(() => {});
   }, []);
 
   async function handleSave() {
@@ -192,6 +198,36 @@ export default function SettingsPage() {
         </div>
       </div>
 
+      <div className="card" style={{ marginBottom: 24 }}>
+        <h3 style={{ marginBottom: 20 }}>Data Sources</h3>
+        <div style={{ fontSize: 13, color: "var(--text-muted)", marginBottom: 16 }}>
+          Configured in <code>config/default.yaml</code>
+        </div>
+        {sources.length === 0 ? (
+          <div style={{ color: "var(--text-muted)", fontSize: 14 }}>No data sources configured.</div>
+        ) : (
+          <div style={{ display: "grid", gap: 12 }}>
+            {sources.map((s) => (
+              <div key={s.name} style={{
+                padding: "12px 16px",
+                background: "var(--bg)",
+                borderRadius: 8,
+                border: "1px solid var(--border)",
+                display: "flex",
+                justifyContent: "space-between",
+                alignItems: "center",
+              }}>
+                <div>
+                  <div style={{ fontWeight: 600, marginBottom: 4 }}>{s.name}</div>
+                  <code style={{ fontSize: 12, color: "var(--text-muted)" }}>{s.url}</code>
+                </div>
+                <span className="badge badge-completed">active</span>
+              </div>
+            ))}
+          </div>
+        )}
+      </div>
+
       <div style={{ display: "flex", alignItems: "center", gap: 16 }}>
         <button className="btn btn-primary" onClick={handleSave} disabled={saving}>
           {saving ? "Saving..." : "Save to .env"}

+ 22 - 0
apps/web/src/lib/config.ts

@@ -0,0 +1,22 @@
+import { readFileSync, existsSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { parse as parseYaml } from "yaml";
+
+const projectRoot = resolve(dirname(new URL(import.meta.url).pathname), "../../../..");
+
+export function loadConfig(configPath?: string): Record<string, any> | null {
+  const paths = [
+    configPath,
+    resolve(projectRoot, "pipeline.config.yaml"),
+    resolve(projectRoot, "pipeline.config.yml"),
+    resolve(projectRoot, "config/default.yaml"),
+  ].filter(Boolean) as string[];
+
+  for (const p of paths) {
+    if (existsSync(p)) {
+      const content = readFileSync(p, "utf-8");
+      return parseYaml(content);
+    }
+  }
+  return null;
+}

+ 2 - 1
apps/web/src/lib/job-store.ts

@@ -8,7 +8,8 @@ export interface JobData {
   input: {
     text: string;
     template: string;
-    platform: string;
+    platform?: string;
+    platforms?: string[];
     ttsProvider: string;
     voiceId?: string;
   };

+ 33 - 14
config/default.yaml

@@ -1,33 +1,42 @@
 # Pipeline default configuration
+#
+# Priority: CLI flags > this file > .env (for secrets/endpoints)
 
 llm:
-  baseURL: "https://api.openai.com/v1"
-  model: "gpt-4o"
-  # apiKey: set via OPENAI_API_KEY env var
+  # baseURL: set via OPENAI_BASE_URL in .env
+  # apiKey: set via OPENAI_API_KEY in .env
+  model: "glm-5.1"
 
 tts:
   provider: "openai-tts"
-  # alignment:                     # Timestamp alignment for subtitles
-  #   provider: "whisper"          # "whisper" = use Whisper forced alignment (recommended)
-  #                                 # "native"  = use TTS provider's built-in timestamps
-  #   whisperModel: "base"         # Model size: tiny/base/small/medium/large
-  #   language: "zh"               # Language hint (auto-detected if omitted)
+  speed: 1.0
+  format: "mp3"
+  # alignment:
+  #   provider: "whisper"
+  #   whisperModel: "base"
+  #   language: "zh"
+
   openai-tts:
-    # apiKey: set via OPENAI_TTS_API_KEY env var
-    # baseURL: defaults to https://new-api.corp.shuidi.tech/v1
+    # apiKey: set via OPENAI_TTS_API_KEY in .env
+    # baseURL: set via OPENAI_TTS_BASE_URL in .env
     model: "seed-tts-1.1"
     defaultVoice: "zh_female_vv_uranus_bigtts"
+
   fish-audio:
-    # apiKey: set via FISH_AUDIO_API_KEY env var
+    # apiKey: set via FISH_AUDIO_API_KEY in .env
     model: "s2-pro"
     defaultVoice: ""
+
   minimax:
-    # apiKey: set via MINIMAX_API_KEY env var
-    groupId: ""
+    # apiKey: set via MINIMAX_API_KEY in .env
+    # groupId: set via MINIMAX_GROUP_ID in .env
     model: "speech-02-hd"
+    defaultVoice: ""
+
   elevenlabs:
-    # apiKey: set via ELEVENLABS_API_KEY env var
+    # apiKey: set via ELEVENLABS_API_KEY in .env
     model: "eleven_multilingual_v2"
+    defaultVoice: "21m00Tcm4TlvDq8ikWAM"
 
 render:
   fps: 30
@@ -50,3 +59,13 @@ templates:
   marketing:
     primaryColor: "#9333ea"
     accentColor: "#ec4899"
+
+collect:
+  github-trending:
+    url: "https://github.crawler.corp.shuidi.tech/api/trending"
+  github-repo:
+    url: "https://github.crawler.corp.shuidi.tech/api/repos/:owner/:repo"
+  github-daily:
+    trendingUrl: "https://github.crawler.corp.shuidi.tech/api/trending"
+    repoUrl: "https://github.crawler.corp.shuidi.tech/api/repos/:owner/:repo"
+    maxRepos: 5

+ 24 - 0
packages/collect/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@pipeline/collect",
+  "version": "0.0.1",
+  "private": true,
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@pipeline/shared": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0"
+  }
+}

+ 96 - 0
packages/collect/src/collectors/github-daily.ts

@@ -0,0 +1,96 @@
+import type { DataSource, CollectResult, CollectParams } from "../types.js";
+import { extractItems } from "../types.js";
+import { registerCollector } from "../registry.js";
+
+class GitHubDailyCollector implements DataSource {
+  readonly name = "github-daily";
+  private trendingUrl: string;
+  private repoUrlTemplate: string;
+  private maxRepos: number;
+
+  constructor(config?: Record<string, any>) {
+    this.trendingUrl = config?.trendingUrl ?? "";
+    this.repoUrlTemplate = config?.repoUrl ?? "";
+    this.maxRepos = config?.maxRepos ?? 5;
+  }
+
+  async collect(_params?: CollectParams): Promise<CollectResult> {
+    if (!this.trendingUrl) {
+      throw new Error("github-daily collector requires 'trendingUrl' in config");
+    }
+    if (!this.repoUrlTemplate) {
+      throw new Error("github-daily collector requires 'repoUrl' in config");
+    }
+
+    // Step 1: fetch trending repos
+    const trendingRes = await fetch(this.trendingUrl);
+    if (!trendingRes.ok) {
+      throw new Error(`Trending API error: ${trendingRes.status} ${await trendingRes.text()}`);
+    }
+    const trendingRaw = await trendingRes.json() as any;
+    const repos = extractItems(trendingRaw).slice(0, this.maxRepos);
+
+    // Step 2: fetch details for each repo
+    const details: string[] = [];
+    for (const repo of repos) {
+      const owner = repo.author ?? repo.owner?.login ?? "";
+      const name = repo.name ?? "";
+      if (!owner || !name) continue;
+
+      const url = this.repoUrlTemplate
+        .replace(":owner", encodeURIComponent(owner))
+        .replace(":repo", encodeURIComponent(name));
+
+      try {
+        const res = await fetch(url);
+        if (!res.ok) continue;
+        const raw = await res.json() as any;
+        const data = raw.data ?? raw;
+        details.push(formatRepo(data));
+      } catch {
+        // Skip failed repo lookups
+      }
+    }
+
+    // Step 3: combine
+    const header = formatTrendingSummary(repos);
+    const content = details.length > 0
+      ? header + "\n\n---\n\n" + details.join("\n\n---\n\n")
+      : header;
+
+    return { type: "text", content };
+  }
+}
+
+function formatTrendingSummary(repos: any[]): string {
+  const lines: string[] = ["# GitHub Trending Today\n"];
+  for (const repo of repos) {
+    const name = repo.fullName || `${repo.author}/${repo.name}`;
+    lines.push(`- **${name}**: ${repo.description ?? ""} (${repo.language ?? ""}, ${repo.stars ?? "?"} stars)`);
+  }
+  return lines.join("\n");
+}
+
+function formatRepo(data: any): string {
+  const name = data.fullName || data.full_name || "unknown";
+  const lines: string[] = [`## ${name}`];
+
+  if (data.description) lines.push(`\n${data.description}`);
+  const meta: string[] = [];
+  if (data.language) meta.push(`Language: ${data.language}`);
+  if (data.stars != null) meta.push(`Stars: ${data.stars}`);
+  else if (data.stargazers_count != null) meta.push(`Stars: ${data.stargazers_count}`);
+  if (data.forks != null) meta.push(`Forks: ${data.forks}`);
+  if (data.topics?.length) meta.push(`Topics: ${data.topics.join(", ")}`);
+  if (meta.length) lines.push(meta.map((m) => `- ${m}`).join("\n"));
+
+  if (data.readme) {
+    // Truncate readme to avoid excessive content
+    const readme = data.readme.length > 2000 ? data.readme.slice(0, 2000) + "\n..." : data.readme;
+    lines.push(`\n### README\n${readme}`);
+  }
+
+  return lines.join("\n");
+}
+
+registerCollector("github-daily", (config) => new GitHubDailyCollector(config));

+ 55 - 0
packages/collect/src/collectors/github-repo.ts

@@ -0,0 +1,55 @@
+import type { DataSource, CollectResult, CollectParams } from "../types.js";
+import { registerCollector } from "../registry.js";
+
+class GitHubRepoCollector implements DataSource {
+  readonly name = "github-repo";
+  private urlTemplate: string;
+
+  constructor(config?: Record<string, any>) {
+    this.urlTemplate = config?.url ?? "";
+  }
+
+  async collect(params?: CollectParams): Promise<CollectResult> {
+    const owner = params?.args?.owner;
+    const repo = params?.args?.repo;
+
+    if (!owner || !repo) {
+      throw new Error("github-repo collector requires --owner and --repo parameters");
+    }
+    if (!this.urlTemplate) {
+      throw new Error("github-repo collector requires 'url' in config");
+    }
+
+    const url = this.urlTemplate
+      .replace(":owner", encodeURIComponent(owner))
+      .replace(":repo", encodeURIComponent(repo));
+
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`GitHub repo API error: ${response.status} ${await response.text()}`);
+    }
+
+    const raw = (await response.json()) as Record<string, any>;
+    const data = (raw.data ?? raw) as Record<string, any>;
+
+    const name = data.fullName || data.full_name || "unknown";
+    const lines: string[] = [`# ${name}\n`];
+    if (data.description) lines.push(`${data.description}\n`);
+    if (data.language) lines.push(`- Language: ${data.language}`);
+    if (data.stars != null) lines.push(`- Stars: ${data.stars}`);
+    if (data.forks != null) lines.push(`- Forks: ${data.forks}`);
+    if (data.openIssues != null) lines.push(`- Open Issues: ${data.openIssues}`);
+    if (data.topics?.length) lines.push(`- Topics: ${data.topics.join(", ")}`);
+    lines.push("");
+    if (data.readme) {
+      const readme = data.readme.length > 3000
+        ? data.readme.slice(0, 3000) + "\n..."
+        : data.readme;
+      lines.push("## README\n", readme);
+    }
+
+    return { type: "text", content: lines.join("\n") };
+  }
+}
+
+registerCollector("github-repo", (config) => new GitHubRepoCollector(config));

+ 42 - 0
packages/collect/src/collectors/github-trending.ts

@@ -0,0 +1,42 @@
+import type { DataSource, CollectResult, CollectParams } from "../types.js";
+import { extractItems } from "../types.js";
+import { registerCollector } from "../registry.js";
+
+class GitHubTrendingCollector implements DataSource {
+  readonly name = "github-trending";
+  private url: string;
+
+  constructor(config?: Record<string, any>) {
+    this.url = config?.url ?? "";
+  }
+
+  async collect(_params?: CollectParams): Promise<CollectResult> {
+    if (!this.url) {
+      throw new Error("github-trending collector requires 'url' in config");
+    }
+
+    const response = await fetch(this.url);
+    if (!response.ok) {
+      throw new Error(`GitHub trending API error: ${response.status} ${await response.text()}`);
+    }
+
+    const raw = await response.json();
+    const items = extractItems(raw);
+
+    const lines: string[] = ["# GitHub Trending\n"];
+    for (const repo of items) {
+      const name = repo.fullName || `${repo.author}/${repo.name}`;
+      lines.push(`## ${name}`);
+      if (repo.description) lines.push(`${repo.description}`);
+      if (repo.language) lines.push(`- Language: ${repo.language}`);
+      if (repo.stars != null) lines.push(`- Stars: ${repo.stars}`);
+      if (repo.currentPeriodStars != null) lines.push(`- Today: +${repo.currentPeriodStars}`);
+      if (repo.url) lines.push(`- URL: ${repo.url}`);
+      lines.push("");
+    }
+
+    return { type: "text", content: lines.join("\n") };
+  }
+}
+
+registerCollector("github-trending", (config) => new GitHubTrendingCollector(config));

+ 7 - 0
packages/collect/src/index.ts

@@ -0,0 +1,7 @@
+export type { DataSource, CollectResult, CollectParams } from "./types.js";
+export { extractItems } from "./types.js";
+export { getCollector, listCollectorNames, registerCollector } from "./registry.js";
+
+import "./collectors/github-trending.js";
+import "./collectors/github-repo.js";
+import "./collectors/github-daily.js";

+ 27 - 0
packages/collect/src/registry.ts

@@ -0,0 +1,27 @@
+import type { DataSource } from "./types.js";
+
+const collectors = new Map<string, (config?: Record<string, any>) => DataSource>();
+
+export function registerCollector(
+  name: string,
+  factory: (config?: Record<string, any>) => DataSource
+): void {
+  collectors.set(name, factory);
+}
+
+export function getCollector(
+  name: string,
+  config?: Record<string, any>
+): DataSource {
+  const factory = collectors.get(name);
+  if (!factory) {
+    throw new Error(
+      `Unknown collector: ${name}. Available: ${[...collectors.keys()].join(", ")}`
+    );
+  }
+  return factory(config);
+}
+
+export function listCollectorNames(): string[] {
+  return [...collectors.keys()];
+}

+ 26 - 0
packages/collect/src/types.ts

@@ -0,0 +1,26 @@
+import type { VideoInput } from "@pipeline/shared";
+
+export type CollectResult =
+  | { type: "text"; content: string }
+  | { type: "json"; content: VideoInput };
+
+export interface CollectParams {
+  args?: Record<string, string>;
+}
+
+export interface DataSource {
+  readonly name: string;
+  collect(params?: CollectParams): Promise<CollectResult>;
+}
+
+/** Extract an array from various API response shapes like { code, data: { list: [...] } } */
+export function extractItems(data: any): any[] {
+  if (Array.isArray(data)) return data;
+  if (data && typeof data === "object") {
+    if (data.data) return extractItems(data.data);
+    for (const key of ["list", "items", "results", "repos"]) {
+      if (Array.isArray(data[key])) return data[key];
+    }
+  }
+  return [];
+}

+ 12 - 0
packages/collect/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src",
+    "types": ["node"]
+  },
+  "include": ["src"],
+  "references": [
+    { "path": "../shared" }
+  ]
+}

+ 43 - 31
packages/core/src/pipeline.ts

@@ -4,6 +4,7 @@ import { join } from "node:path";
 import type {
   PipelineInput,
   PipelineJob,
+  ExportFile,
 } from "@pipeline/shared";
 import { PLATFORM_PRESETS } from "@pipeline/shared";
 import { parseText } from "./stages/parse.js";
@@ -22,6 +23,7 @@ export interface PipelineConfig {
   tts: {
     provider: string;
     voiceId?: string;
+    model?: string;
     format?: "mp3" | "wav" | "pcm";
     speed?: number;
   };
@@ -59,6 +61,7 @@ export async function runPipeline(
   const workDir = join(config.output.dir, "tmp", jobId);
   await mkdir(workDir, { recursive: true });
 
+  const platforms = input.platforms;
   const job: PipelineJob = {
     id: jobId,
     input,
@@ -68,19 +71,21 @@ export async function runPipeline(
   };
 
   try {
-    // Stage 1: Parse
+    // Stage 1: Parse (shared across all platforms)
     callbacks?.onStageStart?.("parse");
     const parsed = await parseText(input.text, input.template, {
       llm: config.llm,
+      skipLlm: config.skipLlm,
     });
     job.parsed = parsed;
     callbacks?.onStageComplete?.("parse");
 
-    // Stage 2: TTS (can be skipped with --no-tts)
+    // Stage 2: TTS (shared across all platforms)
     callbacks?.onStageStart?.("tts");
     const tts = await generateTTS(parsed, workDir, {
-      provider: input.ttsProvider,
-      voiceId: input.voiceId,
+      provider: config.tts.provider,
+      voiceId: config.tts.voiceId || input.voiceId,
+      model: config.tts.model,
       format: config.tts.format,
       speed: config.tts.speed,
       skip: config.skipTts,
@@ -89,37 +94,44 @@ export async function runPipeline(
     job.tts = tts;
     callbacks?.onStageComplete?.("tts");
 
-    // Stage 3: Assets
-    callbacks?.onStageStart?.("assets");
-    const assets = await resolveAssets(
-      parsed, workDir, config.assets.root, config.assets.inputDir,
-      { template: input.template, aspect: PLATFORM_PRESETS[input.platform].aspect }
-    );
-    callbacks?.onStageComplete?.("assets");
+    // Stages 3-6: Per-platform loop
+    const allExportFiles: ExportFile[] = [];
+    job.composed = {};
 
-    // Stage 4: Compose
-    callbacks?.onStageStart?.("compose");
-    const composed = composeProject(parsed, tts, assets, input.platform, input.template);
-    job.composed = composed;
-    callbacks?.onStageComplete?.("compose");
+    for (const platform of platforms) {
+      // Stage 3: Assets
+      callbacks?.onStageStart?.(`assets:${platform}`);
+      const assets = await resolveAssets(
+        parsed, workDir, config.assets.root, config.assets.inputDir,
+        { template: input.template, aspect: PLATFORM_PRESETS[platform].aspect }
+      );
+      callbacks?.onStageComplete?.(`assets:${platform}`);
 
-    // Stage 5: Render
-    callbacks?.onStageStart?.("render");
-    const renderOutput = join(workDir, "render.mp4");
-    await renderVideo(composed, renderOutput, config.templates.entryPoint);
-    callbacks?.onStageComplete?.("render");
+      // Stage 4: Compose
+      callbacks?.onStageStart?.(`compose:${platform}`);
+      const composed = composeProject(parsed, tts, assets, platform, input.template);
+      job.composed[platform] = composed;
+      callbacks?.onStageComplete?.(`compose:${platform}`);
 
-    // Stage 6: Export
-    callbacks?.onStageStart?.("export");
-    const exported = await exportVideo(
-      renderOutput,
-      composed,
-      config.output.dir,
-      jobId
-    );
-    job.exported = exported;
-    callbacks?.onStageComplete?.("export");
+      // Stage 5: Render
+      callbacks?.onStageStart?.(`render:${platform}`);
+      const renderOutput = join(workDir, `render-${platform}.mp4`);
+      await renderVideo(composed, renderOutput, config.templates.entryPoint);
+      callbacks?.onStageComplete?.(`render:${platform}`);
 
+      // Stage 6: Export
+      callbacks?.onStageStart?.(`export:${platform}`);
+      const exported = await exportVideo(
+        renderOutput,
+        composed,
+        config.output.dir,
+        jobId
+      );
+      allExportFiles.push(...exported.files);
+      callbacks?.onStageComplete?.(`export:${platform}`);
+    }
+
+    job.exported = { jobId, createdAt: Date.now(), files: allExportFiles };
     job.status = "completed";
   } catch (err) {
     job.status = "failed";

+ 63 - 8
packages/core/src/stages/parse.ts

@@ -1,4 +1,10 @@
-import { VideoInputSchema, ParsedContentSchema } from "@pipeline/shared";
+import {
+  VideoInputSchema,
+  ParsedContentSchema,
+  detectInputFormat,
+  LLMClient,
+  getParsePrompt,
+} from "@pipeline/shared";
 import type { ParsedContent, Scene, VideoInput } from "@pipeline/shared";
 
 export interface ParseStageConfig {
@@ -7,15 +13,58 @@ export interface ParseStageConfig {
     apiKey?: string;
     model: string;
   };
+  skipLlm?: boolean;
 }
 
 export async function parseText(
   text: string,
-  _template: string,
-  _config: ParseStageConfig
+  template: string,
+  config: ParseStageConfig
 ): Promise<ParsedContent> {
-  const parsed = JSON.parse(text);
-  const videoInput: VideoInput = VideoInputSchema.parse(parsed);
+  // --- Input normalization ---
+  const detected = detectInputFormat(text);
+
+  let videoInput: VideoInput;
+
+  if (detected.format === "valid-schema") {
+    videoInput = detected.parsed!;
+  } else if (
+    config.skipLlm ||
+    !(config.llm.apiKey || process.env.OPENAI_API_KEY)
+  ) {
+    throw new Error(
+      `Input is not valid VideoInputSchema JSON and AI processing is disabled. ` +
+        `Provide structured JSON matching VideoInputSchema or enable LLM processing (set OPENAI_API_KEY or remove --skip-llm).`
+    );
+  } else {
+    const client = new LLMClient(config.llm);
+    const systemPrompt = getParsePrompt(template as "news" | "knowledge" | "opinion" | "marketing");
+    const rawResponse = await client.chat(systemPrompt, text);
+
+    // Strip markdown code block wrapper if present (```json ... ```)
+    const cleaned = rawResponse
+      .replace(/^```(?:json)?\s*\n?/i, "")
+      .replace(/\n?```\s*$/i, "")
+      .trim();
+
+    let aiParsed: unknown;
+    try {
+      aiParsed = JSON.parse(cleaned);
+    } catch {
+      throw new Error(
+        `AI returned invalid JSON: ${cleaned.slice(0, 200)}`
+      );
+    }
+
+    const validationResult = VideoInputSchema.safeParse(aiParsed);
+    if (!validationResult.success) {
+      throw new Error(
+        `AI output does not match VideoInputSchema: ${validationResult.error.message}`
+      );
+    }
+    videoInput = validationResult.data;
+  }
+  // --- End input normalization ---
 
   const scenes: Scene[] = [];
   let sceneIndex = 0;
@@ -30,9 +79,15 @@ export async function parseText(
       title: videoInput.title,
       keyframes: videoInput.cover.keyframes ?? [],
       images: [
-        ...(videoInput.cover.imagePath ? [{ path: videoInput.cover.imagePath }] : []),
-        ...(videoInput.cover.imageUrl ? [{ url: videoInput.cover.imageUrl }] : []),
-        ...(videoInput.cover.imageQuery ? [{ query: videoInput.cover.imageQuery }] : []),
+        ...(videoInput.cover.imagePath
+          ? [{ path: videoInput.cover.imagePath }]
+          : []),
+        ...(videoInput.cover.imageUrl
+          ? [{ url: videoInput.cover.imageUrl }]
+          : []),
+        ...(videoInput.cover.imageQuery
+          ? [{ query: videoInput.cover.imageQuery }]
+          : []),
       ],
       backgroundQuery: videoInput.cover.imageQuery,
     });

+ 15 - 0
packages/core/src/stages/tts.ts

@@ -7,6 +7,7 @@ import { join } from "node:path";
 export interface TTSStageConfig {
   provider: string;
   voiceId?: string;
+  model?: string;
   format?: "mp3" | "wav" | "pcm";
   speed?: number;
   skip?: boolean;
@@ -33,6 +34,20 @@ export async function generateTTS(
   const provider = getProvider(config.provider);
   const globalSpeed = config.speed ?? 1.0;
   const useWhisper = config.alignment?.provider === "whisper";
+
+  // Inject model name as env var so providers pick it up alongside their env-based config
+  if (config.model) {
+    const envMap: Record<string, string> = {
+      "openai-tts": "OPENAI_TTS_MODEL",
+      "fish-audio": "FISH_AUDIO_MODEL",
+      "minimax": "MINIMAX_MODEL",
+      "elevenlabs": "ELEVENLABS_MODEL",
+    };
+    const envKey = envMap[config.provider];
+    if (envKey && !process.env[envKey]) {
+      process.env[envKey] = config.model;
+    }
+  }
   const sceneResults: TTSSceneResult[] = [];
   let currentOffset = 0;
 

+ 2 - 2
packages/shared/src/index.ts

@@ -12,5 +12,5 @@ export type {
   Pace,
   JobStatus,
 } from "./constants.js";
-export { LLMClient, getParsePrompt } from "./llm/index.js";
-export type { LLMClientConfig } from "./llm/index.js";
+export { LLMClient, getParsePrompt, detectInputFormat } from "./llm/index.js";
+export type { LLMClientConfig, InputFormat, DetectResult } from "./llm/index.js";

+ 25 - 0
packages/shared/src/llm/detect-input.ts

@@ -0,0 +1,25 @@
+import { VideoInputSchema } from "../types/scene.js";
+import type { VideoInput } from "../types/scene.js";
+
+export type InputFormat = "valid-schema" | "invalid-json" | "non-json";
+
+export interface DetectResult {
+  format: InputFormat;
+  parsed?: VideoInput;
+}
+
+export function detectInputFormat(text: string): DetectResult {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return { format: "non-json" };
+  }
+
+  const result = VideoInputSchema.safeParse(parsed);
+  if (result.success) {
+    return { format: "valid-schema", parsed: result.data };
+  }
+
+  return { format: "invalid-json" };
+}

+ 2 - 0
packages/shared/src/llm/index.ts

@@ -1,3 +1,5 @@
 export { LLMClient } from "./client.js";
 export type { LLMClientConfig } from "./client.js";
 export { getParsePrompt } from "./prompts/parse-text.js";
+export { detectInputFormat } from "./detect-input.js";
+export type { InputFormat, DetectResult } from "./detect-input.js";

+ 3 - 3
packages/shared/src/types/pipeline.ts

@@ -4,13 +4,13 @@ import { WordTimestampSchema, SceneImageSchema, KeyframeSchema, ParsedContentSch
 export const PipelineInputSchema = z.object({
   text: z.string(),
   template: z.enum(["news", "knowledge", "opinion", "marketing"]),
-  platform: z.enum([
+  platforms: z.array(z.enum([
     "bilibili",
     "douyin-long",
     "douyin-short",
     "universal-16x9",
     "universal-9x16",
-  ]),
+  ])).min(1),
   ttsProvider: z.enum(["fish-audio", "minimax", "elevenlabs", "openai-tts"]),
   voiceId: z.string().optional(),
   outputPath: z.string().optional(),
@@ -128,7 +128,7 @@ export const PipelineJobSchema = z.object({
   error: z.string().optional(),
   parsed: ParsedContentSchema.optional(),
   tts: TTSStageResultSchema.optional(),
-  composed: ComposedProjectSchema.optional(),
+  composed: z.record(z.string(), ComposedProjectSchema).optional(),
   exported: ExportManifestSchema.optional(),
   createdAt: z.number(),
   updatedAt: z.number(),

+ 5 - 34
packages/templates/src/base/components/subtitle-bar.tsx

@@ -1,5 +1,5 @@
 import React from "react";
-import { useCurrentFrame, useVideoConfig, interpolate } from "remotion";
+import { useCurrentFrame, useVideoConfig } from "remotion";
 import type { WordTimestamp } from "@pipeline/shared";
 
 interface SubtitleBarProps {
@@ -57,19 +57,6 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
 
   if (!currentSegment) return null;
 
-  let currentWordIndex = -1;
-  for (let i = 0; i < currentSegment.words.length; i++) {
-    if (
-      currentTime >= currentSegment.words[i].startSeconds &&
-      currentTime <= currentSegment.words[i].endSeconds
-    ) {
-      currentWordIndex = i;
-      break;
-    }
-  }
-
-  const fadeIn = interpolate(frame, [0, 6], [0, 1], { extrapolateRight: "clamp" });
-
   return (
     <div
       style={{
@@ -79,7 +66,6 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
         right: 0,
         display: "flex",
         justifyContent: "center",
-        opacity: fadeIn,
         padding: "0 5%",
         ...style,
       }}
@@ -97,6 +83,7 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
       >
         <span
           style={{
+            color: "#ffffff",
             fontSize: 40,
             lineHeight: 1.6,
             fontFamily: "sans-serif",
@@ -108,25 +95,9 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
         >
           {currentSegment.words
             .filter((w) => !/[。,!?\s]/.test(w.word))
-            .map((w, i) => {
-            const isCurrent = i === currentWordIndex;
-            const isPast = currentTime > w.endSeconds;
-            return (
-              <span
-                key={i}
-                style={{
-                  color: isCurrent
-                    ? "#fbbf24"
-                    : isPast
-                      ? "rgba(255,255,255,0.5)"
-                      : "#ffffff",
-                  fontWeight: isCurrent ? 700 : 400,
-                }}
-              >
-                {w.word}
-              </span>
-            );
-          })}
+            .map((w, i) => (
+              <span key={i}>{w.word}</span>
+          ))}
         </span>
       </div>
     </div>

+ 1 - 1
packages/tts/src/providers/openai-tts.ts

@@ -31,7 +31,7 @@ class OpenAITTSProvider implements TTSProvider {
         model,
         response_format: request.format || "mp3",
         speed: request.speed ?? 1.0,
-        voice: request.voiceId || "zh_female_vv_uranus_bigtts",
+        voice: request.voiceId || "zh_female_linjianvhai_uranus_bigtts",
       }),
     });
 

+ 19 - 0
pnpm-lock.yaml

@@ -17,6 +17,9 @@ importers:
 
   apps/cli:
     dependencies:
+      '@pipeline/collect':
+        specifier: workspace:*
+        version: link:../../packages/collect
       '@pipeline/core':
         specifier: workspace:*
         version: link:../../packages/core
@@ -51,6 +54,9 @@ importers:
 
   apps/web:
     dependencies:
+      '@pipeline/collect':
+        specifier: workspace:*
+        version: link:../../packages/collect
       '@pipeline/shared':
         specifier: workspace:*
         version: link:../../packages/shared
@@ -63,6 +69,9 @@ importers:
       react-dom:
         specifier: ^19.0.0
         version: 19.2.6(react@19.2.6)
+      yaml:
+        specifier: ^2.7.0
+        version: 2.9.0
       zod:
         specifier: ^3.24.0
         version: 3.25.76
@@ -80,6 +89,16 @@ importers:
         specifier: ^5.8.0
         version: 5.9.3
 
+  packages/collect:
+    dependencies:
+      '@pipeline/shared':
+        specifier: workspace:*
+        version: link:../shared
+    devDependencies:
+      '@types/node':
+        specifier: ^22.0.0
+        version: 22.19.19
+
   packages/core:
     dependencies:
       '@pipeline/shared':

+ 73 - 0
test/fixtures/test-news.md

@@ -0,0 +1,73 @@
+## 热点信息速报 — 2026年6月2日
+
+### 1. 中国科技与互联网公司
+
+**DeepSeek V4 正式发布,引爆国产AI芯片需求**
+- DeepSeek V4 于2026年4月24日开源发布,支持原生百万Token上下文窗口
+- DeepSeek 计划融资500亿元,估值或破3500亿元,将创中国AI企业史上最大单笔融资纪录
+- 华为昇腾950系列AI芯片需求因此大幅飙升
+
+**字节跳动、腾讯、阿里巴巴竞相采购华为昇腾芯片**
+- 三大互联网巨头向华为批量订购数十万颗昇腾950PR芯片
+- 美方出口限制与中方进口门槛双重压力下,国产AI算力自主化加速推进
+- 字节跳动因版权纠纷暂停全球视频生成项目部分业务
+
+### 2. 大模型与 AI 产品
+
+(Web搜索受API限流影响,部分结果缺失。以下为可确认信息)
+- DeepSeek V4 开源,百万Token上下文,性能大幅跃升
+- 华为昇腾已同步支持DeepSeek系列模型,通过芯模协同实现全系列产品适配
+
+### 3. AI 基础设施与云算力
+
+**NVIDIA 持续主导,进军PC市场**
+- NVIDIA 占据AI加速器市场约80%份额,FY2026数据中心收入$1937亿
+- 发布 RTX Spark superchip,进军消费级AI PC市场,Dell/HP/Surface 预计6月初推出首批产品
+- NVIDIA GTC 2026 聚焦Agentic AI、推理和Physical AI突破
+
+**AMD 努力追赶**
+- AMD 占AI加速器市场约5-7%(~$70-80亿数据中心销售)
+- Advancing AI 2026 大会将于7月22-23日在旧金山举行
+
+### 4. 智能汽车与自动驾驶
+
+**特斯拉FSD入华**
+- 2026年5月官宣监督版FSD可在中国使用(L2级,依据联合国R-171法规)
+- FSD入华预计2026年8月左右大规模落地
+- FSD V14采用千亿参数模型,但HW3硬件无法支持无监督自动驾驶
+
+**比亚迪智驾进展**
+- 璇玑A3芯片纸面参数强大,但实际表现仍待验证
+- 泊车功能使用率从21%飙升至93%,事故率几乎降为零
+- 分析师认为比亚迪整体架构与特斯拉仍有差距
+
+### 5. 机器人与具身智能
+
+**2026年中国人形机器人产量同比增长94%,接近翻倍**
+- 宇树+智元合计占据国内约80%出货份额,形成双寡头格局
+- 智元:远征A3达成10,000台下线,2025年出货5,168台(行业第一)
+- 宇树:2025年出货5,500台(全球第一),申请科创板IPO,毛利率60%
+- 宇树首发「人形机器人App Store」,但73.6%出货仍为科研客户
+- 头部企业2026年合计年产能预期5-10万台
+
+### 6. 全球科技巨头
+
+- NVIDIA RTX Spark superchip发布,联手Microsoft/Dell/HP进军AI PC
+- AMD Advancing AI 2026大会7月举办,路线图对标NVIDIA
+
+### 8. 地缘政治与国际关系
+
+**中美"竞争中有接触"新阶段**
+- 特朗普2026年5月13-15日对中国进行国事访问
+- 美国就部分中国商品适用较低关税征求意见,释放一定积极信号
+- 美国计划2027年6月起对中国半导体加征新关税(18个月过渡期),中方强硬回击
+- 美国多线强势介入引发全球地缘风险,中欧关系可能迎来窗口期
+
+### 10. 能源与电力系统
+
+**光伏首次成为全球供能增量最大来源**
+- 2026年4月中国单月新增光伏项目5,651个
+- 核电从"补充选项"升级为"稳定支柱",发电量创历史新高
+- 预计到2030年,可再生能源与核电合计提供全球50%电力
+- 中国推进光伏行业"内卷式"竞争整治,头部企业毛利率有望回升超5%
+