修复字幕与音频不同步的问题

apps/cli/src/commands/render.ts

@@ -19,6 +19,9 @@ export const renderCommand = new Command("render")
   .option("--assets-dir <dir>", "Image assets directory (defaults to JSON file location)")
   .option("--config <path>", "Config file path")
   .option("--no-tts", "Skip TTS (generate silent video)")
+  .option("--alignment <mode>", "Timestamp alignment: whisper|native")
+  .option("--whisper-model <model>", "Whisper model size (tiny/base/small/medium/large)", "base")
+  .option("--alignment-language <lang>", "Language hint for whisper (e.g. zh, en)")
   .option("-v, --verbose", "Verbose logging")
   .action(async (input, opts) => {
     const template = opts.template as string;
@@ -79,6 +82,7 @@ export const renderCommand = new Command("render")
       assets: { root: assetsRoot, inputDir },
       templates: { entryPoint: templatesEntry },
       skipTts: noTts,
+      alignment: buildAlignmentConfig(opts, config),
     };
 
     const stages = ["parse", "tts", "assets", "compose", "render", "export"];
@@ -100,7 +104,7 @@ export const renderCommand = new Command("render")
         onStageStart: (stage) => {
           const labels: Record<string, string> = {
             parse: "Parsing JSON input",
-            tts: noTts ? "Generating silent audio" : "Generating narration",
+            tts: noTts ? "Generating silent audio" : `Generating narration${pipelineConfig.alignment?.provider === "whisper" ? " (whisper alignment)" : ""}`,
             assets: "Resolving assets",
             compose: "Composing scenes",
             render: "Rendering video",
@@ -132,6 +136,19 @@ export const renderCommand = new Command("render")
     }
   });
 
+function buildAlignmentConfig(
+  opts: any,
+  config: Record<string, any> | null
+): PipelineConfig["alignment"] {
+  const mode = opts.alignment || config?.alignment?.provider;
+  if (mode !== "whisper" && mode !== "native") return undefined;
+  return {
+    provider: mode,
+    whisperModel: opts.whisperModel || config?.alignment?.whisperModel || "base",
+    language: opts.alignmentLanguage || config?.alignment?.language,
+  };
+}
+
 function loadConfig(configPath?: string): Record<string, any> | null {
   const paths = [
     configPath,

config/default.yaml

@@ -7,6 +7,11 @@ llm:
 
 tts:
   provider: "openai-tts"
+  # alignment:                     # Timestamp alignment for subtitles
+  #   provider: "whisper"          # "whisper" = use Whisper forced alignment (recommended)
+  #                                 # "native"  = use TTS provider's built-in timestamps
+  #   whisperModel: "base"         # Model size: tiny/base/small/medium/large
+  #   language: "zh"               # Language hint (auto-detected if omitted)
   openai-tts:
     # apiKey: set via OPENAI_TTS_API_KEY env var
     # baseURL: defaults to https://new-api.corp.shuidi.tech/v1

packages/core/src/pipeline.ts

@@ -25,6 +25,11 @@ export interface PipelineConfig {
     format?: "mp3" | "wav" | "pcm";
     speed?: number;
   };
+  alignment?: {
+    provider: "whisper" | "native";
+    whisperModel?: string;
+    language?: string;
+  };
   output: {
     dir: string;
   };
@@ -79,6 +84,7 @@ export async function runPipeline(
       format: config.tts.format,
       speed: config.tts.speed,
       skip: config.skipTts,
+      alignment: config.alignment,
     });
     job.tts = tts;
     callbacks?.onStageComplete?.("tts");

packages/core/src/stages/compose.ts

@@ -29,13 +29,8 @@ export function composeProject(
       ? { id: assets.backgrounds[i].id, localPath: assets.backgrounds[i].localPath }
       : undefined;
 
-    // Convert word timestamps from global offsets to scene-relative time
-    const sceneStartTime = ttsScene?.startOffsetSeconds ?? 0;
-    const wordTimestamps = (ttsScene?.wordTimestamps ?? []).map((wt) => ({
-      word: wt.word,
-      startSeconds: wt.startSeconds - sceneStartTime,
-      endSeconds: wt.endSeconds - sceneStartTime,
-    }));
+    // wordTimestamps are already scene-relative (0-based) from the TTS stage
+    const wordTimestamps = ttsScene?.wordTimestamps ?? [];
 
     return {
       id: scene.id,

packages/core/src/stages/tts.ts

@@ -1,5 +1,5 @@
 import type { TTSStageResult, TTSSceneResult, ParsedContent } from "@pipeline/shared";
-import { getProvider } from "@pipeline/tts";
+import { getProvider, alignWithWhisper } from "@pipeline/tts";
 import { mkdir, writeFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
@@ -10,6 +10,11 @@ export interface TTSStageConfig {
   format?: "mp3" | "wav" | "pcm";
   speed?: number;
   skip?: boolean;
+  alignment?: {
+    provider: "whisper" | "native";
+    whisperModel?: string;
+    language?: string;
+  };
 }
 
 export async function generateTTS(
@@ -27,6 +32,7 @@ export async function generateTTS(
 
   const provider = getProvider(config.provider);
   const globalSpeed = config.speed ?? 1.0;
+  const useWhisper = config.alignment?.provider === "whisper";
   const sceneResults: TTSSceneResult[] = [];
   let currentOffset = 0;
 
@@ -67,12 +73,27 @@ export async function generateTTS(
       ? Math.max(scene.duration, audioDuration)
       : audioDuration;
 
+    // Run whisper alignment when configured, otherwise use provider timestamps
+    let wordTimestamps = result.wordTimestamps;
+
+    if (useWhisper && result.audioFilePath) {
+      const aligned = await alignWithWhisper({
+        audioFilePath: result.audioFilePath,
+        text: scene.narration,
+        model: config.alignment?.whisperModel,
+        language: config.alignment?.language,
+      });
+      if (aligned) {
+        wordTimestamps = aligned;
+      }
+    }
+
     sceneResults.push({
       sceneId: scene.id,
       audioFilePath: result.audioFilePath,
       durationSeconds: sceneDuration,
       startOffsetSeconds: currentOffset,
-      wordTimestamps: result.wordTimestamps,
+      wordTimestamps,
     });
 
     currentOffset += sceneDuration;
@@ -115,8 +136,8 @@ async function generateSilentTTS(
       startOffsetSeconds: currentOffset,
       wordTimestamps: words.map((word, i) => ({
         word,
-        startSeconds: currentOffset + i * durationPerWord,
-        endSeconds: currentOffset + (i + 1) * durationPerWord,
+        startSeconds: i * durationPerWord,
+        endSeconds: (i + 1) * durationPerWord,
       })),
     });
 

packages/templates/src/base/components/subtitle-bar.tsx

@@ -7,6 +7,40 @@ interface SubtitleBarProps {
   style?: React.CSSProperties;
 }
 
+interface Segment {
+  words: WordTimestamp[];
+  startSeconds: number;
+  endSeconds: number;
+}
+
+function groupIntoSegments(wordTimestamps: WordTimestamp[]): Segment[] {
+  if (wordTimestamps.length === 0) return [];
+
+  const segmentEnds = new Set<number>();
+  for (let i = 0; i < wordTimestamps.length; i++) {
+    const w = wordTimestamps[i].word;
+    if (/[。！？；]/.test(w)) {
+      segmentEnds.add(i);
+    } else if (/[，、,;]/.test(w) && i - (segmentEnds.size > 0 ? [...segmentEnds].pop()! : -1) >= 8) {
+      segmentEnds.add(i);
+    }
+  }
+  segmentEnds.add(wordTimestamps.length - 1);
+
+  const segments: Segment[] = [];
+  let segStart = 0;
+  for (const end of [...segmentEnds].sort((a, b) => a - b)) {
+    const words = wordTimestamps.slice(segStart, end + 1);
+    segments.push({
+      words,
+      startSeconds: words[0].startSeconds,
+      endSeconds: words[words.length - 1].endSeconds,
+    });
+    segStart = end + 1;
+  }
+  return segments;
+}
+
 export const SubtitleBar: React.FC<SubtitleBarProps> = ({
   wordTimestamps,
   style,
@@ -15,48 +49,26 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
   const { fps } = useVideoConfig();
   const currentTime = frame / fps;
 
-  // Find current word index
+  const segments = React.useMemo(() => groupIntoSegments(wordTimestamps), [wordTimestamps]);
+
+  const currentSegment = segments.find(
+    (s) => currentTime >= s.startSeconds && currentTime <= s.endSeconds
+  ) ?? segments[segments.length - 1];
+
+  if (!currentSegment) return null;
+
   let currentWordIndex = -1;
-  for (let i = 0; i < wordTimestamps.length; i++) {
+  for (let i = 0; i < currentSegment.words.length; i++) {
     if (
-      currentTime >= wordTimestamps[i].startSeconds &&
-      currentTime <= wordTimestamps[i].endSeconds
+      currentTime >= currentSegment.words[i].startSeconds &&
+      currentTime <= currentSegment.words[i].endSeconds
     ) {
       currentWordIndex = i;
       break;
     }
   }
 
-  // Dynamic window size: fewer words when there's lots of text
-  const totalWords = wordTimestamps.length;
-  const windowSize =
-    totalWords > 50 ? 8 : totalWords > 30 ? 12 : 16;
-
-  const start = Math.max(0, currentWordIndex - Math.floor(windowSize / 2));
-  const end = Math.min(wordTimestamps.length, start + windowSize);
-  const visibleWords = wordTimestamps.slice(start, end).map((w, i) => ({
-    text: w.word,
-    globalIndex: start + i,
-  }));
-
-  const opacity = interpolate(frame, [0, 8], [0, 1], {
-    extrapolateRight: "clamp",
-  });
-
-  // Continuous font scaling based on total word count
-  const baseFontSize = 42;
-  const minScale = 0.5;
-  const scaleAt50 = 0.65;
-  const fontSize =
-    totalWords <= 15
-      ? baseFontSize
-      : totalWords <= 50
-        ? baseFontSize * (1 - (1 - scaleAt50) * ((totalWords - 15) / 35))
-        : baseFontSize * (minScale + (scaleAt50 - minScale) * Math.max(0, 1 - (totalWords - 50) / 100));
-
-  // Dynamic max height: smaller when text is dense
-  const maxHeight =
-    totalWords > 50 ? 80 : totalWords > 30 ? 100 : 120;
+  const fadeIn = interpolate(frame, [0, 6], [0, 1], { extrapolateRight: "clamp" });
 
   return (
     <div
@@ -67,7 +79,7 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
         right: 0,
         display: "flex",
         justifyContent: "center",
-        opacity,
+        opacity: fadeIn,
         padding: "0 5%",
         ...style,
       }}
@@ -76,10 +88,8 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
         style={{
           backgroundColor: "rgba(0,0,0,0.8)",
           borderRadius: 12,
-          padding: "10px 24px",
-          maxWidth: "100%",
-          maxHeight,
-          overflow: "hidden",
+          padding: "14px 28px",
+          maxWidth: "90%",
           display: "flex",
           alignItems: "center",
           justifyContent: "center",
@@ -87,8 +97,8 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
       >
         <span
           style={{
-            fontSize,
-            lineHeight: 1.3,
+            fontSize: 40,
+            lineHeight: 1.6,
             fontFamily: "sans-serif",
             display: "-webkit-box",
             WebkitLineClamp: 2,
@@ -96,12 +106,12 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
             overflow: "hidden",
           }}
         >
-          {visibleWords.map((w, i) => {
-            const isCurrent = w.globalIndex === currentWordIndex;
-            const isPast = currentTime > wordTimestamps[w.globalIndex].endSeconds;
+          {currentSegment.words.map((w, i) => {
+            const isCurrent = i === currentWordIndex;
+            const isPast = currentTime > w.endSeconds;
             return (
               <span
-                key={w.globalIndex}
+                key={i}
                 style={{
                   color: isCurrent
                     ? "#fbbf24"
@@ -111,7 +121,7 @@ export const SubtitleBar: React.FC<SubtitleBarProps> = ({
                   fontWeight: isCurrent ? 700 : 400,
                 }}
               >
-                {w.text}
+                {w.word}
               </span>
             );
           })}

packages/tts/scripts/align.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Whisper-based forced alignment for TTS-generated audio.
+
+Uses faster-whisper (preferred) or openai-whisper to produce word-level
+timestamps. Outputs JSON to stdout.
+
+Usage:
+    python3 align.py <audio_path> [--text TEXT] [--language LANG] [--model MODEL]
+
+Output format:
+    {"ok": true, "words": [{"word": "...", "start": 0.0, "end": 0.5}, ...]}
+    {"ok": false, "error": "error message"}
+"""
+
+import sys
+import json
+import argparse
+
+
+def transcribe_with_faster_whisper(audio_path, language=None, model_size="base"):
+    """Transcribe using faster-whisper and return word-level timestamps."""
+    from faster_whisper import WhisperModel
+
+    model = WhisperModel(model_size, device="cpu", compute_type="int8")
+
+    segments, _info = model.transcribe(
+        audio_path,
+        word_timestamps=True,
+        language=language,
+        vad_filter=True,
+    )
+
+    words = []
+    for segment in segments:
+        for word_info in segment.words:
+            words.append({
+                "word": word_info.word.strip(),
+                "start": round(word_info.start, 3),
+                "end": round(word_info.end, 3),
+            })
+
+    return words
+
+
+def transcribe_with_whisper(audio_path, language=None, model_size="base"):
+    """Fallback: transcribe using openai-whisper and return word-level timestamps."""
+    import whisper
+
+    model = whisper.load_model(model_size)
+    result = model.transcribe(
+        audio_path,
+        word_timestamps=True,
+        language=language,
+    )
+
+    words = []
+    for segment in result.get("segments", []):
+        for word_info in segment.get("words", []):
+            words.append({
+                "word": word_info["word"].strip(),
+                "start": round(word_info["start"], 3),
+                "end": round(word_info["end"], 3),
+            })
+
+    return words
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Whisper forced alignment")
+    parser.add_argument("audio_path", help="Path to audio file")
+    parser.add_argument("--text", default=None, help="Reference text (unused, reserved)")
+    parser.add_argument("--language", default=None, help="Language hint (e.g. 'zh', 'en')")
+    parser.add_argument("--model", default="base", help="Whisper model size (tiny/base/small/medium/large)")
+    args = parser.parse_args()
+
+    try:
+        try:
+            words = transcribe_with_faster_whisper(
+                args.audio_path,
+                language=args.language,
+                model_size=args.model,
+            )
+        except ImportError:
+            words = transcribe_with_whisper(
+                args.audio_path,
+                language=args.language,
+                model_size=args.model,
+            )
+
+        print(json.dumps({"ok": True, "words": words}))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

packages/tts/src/index.ts

@@ -1,5 +1,6 @@
 export type { TTSProvider } from "./types.js";
 export { getProvider, listProviderNames, registerProvider } from "./registry.js";
+export { alignWithWhisper } from "./whisper-align.js";
 
 // Auto-register all providers
 import "./providers/fish-audio.js";

packages/tts/src/whisper-align.ts

@@ -0,0 +1,145 @@
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+import type { WordTimestamp } from "@pipeline/shared";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const ALIGN_SCRIPT = join(__dirname, "..", "scripts", "align.py");
+
+export interface WhisperAlignOptions {
+  audioFilePath: string;
+  text?: string;
+  language?: string;
+  model?: string;
+  timeout?: number;
+}
+
+interface WhisperWord {
+  word: string;
+  start: number;
+  end: number;
+}
+
+interface AlignResult {
+  ok: boolean;
+  words?: WhisperWord[];
+  error?: string;
+}
+
+/**
+ * Run Whisper forced alignment on a TTS-generated audio file.
+ * Returns word-level timestamps mapped to the original narration text.
+ * Falls back to null if whisper is unavailable.
+ */
+export async function alignWithWhisper(
+  options: WhisperAlignOptions
+): Promise<WordTimestamp[] | null> {
+  const args = [
+    ALIGN_SCRIPT,
+    options.audioFilePath,
+    ...(options.language ? ["--language", options.language] : []),
+    ...(options.model ? ["--model", options.model] : []),
+  ];
+
+  const result = await runPython(args, options.timeout ?? 120_000);
+  if (!result.ok || !result.words || result.words.length === 0) {
+    return null;
+  }
+
+  return mapWhisperToOriginal(result.words, options.text);
+}
+
+function runPython(args: string[], timeout: number): Promise<AlignResult> {
+  return new Promise((resolve) => {
+    const proc = spawn("python3", args, {
+      timeout,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+    proc.stdout.on("data", (d: Buffer) => (stdout += d));
+    proc.stderr.on("data", (d: Buffer) => (stderr += d));
+
+    proc.on("close", () => {
+      try {
+        resolve(JSON.parse(stdout) as AlignResult);
+      } catch {
+        resolve({ ok: false, error: stderr || stdout || "parse error" });
+      }
+    });
+
+    proc.on("error", (err) => {
+      resolve({ ok: false, error: err.message });
+    });
+  });
+}
+
+/**
+ * Map whisper's transcription timestamps back to the original text's words.
+ *
+ * Strategy: build a character-to-time mapping from whisper output, then
+ * interpolate each original word/character into that timeline. This works
+ * well for TTS audio where the spoken content closely matches the script.
+ */
+function mapWhisperToOriginal(
+  whisperWords: WhisperWord[],
+  originalText?: string
+): WordTimestamp[] {
+  if (!originalText || whisperWords.length === 0) {
+    return whisperWords.map((w) => ({
+      word: w.word,
+      startSeconds: w.start,
+      endSeconds: w.end,
+    }));
+  }
+
+  // Build character-level time array from whisper output
+  const charTimes: number[] = [];
+  for (const w of whisperWords) {
+    const perChar = (w.end - w.start) / Math.max(w.word.length, 1);
+    for (let i = 0; i < w.word.length; i++) {
+      charTimes.push(w.start + i * perChar);
+    }
+  }
+
+  if (charTimes.length === 0) {
+    return whisperWords.map((w) => ({
+      word: w.word,
+      startSeconds: w.start,
+      endSeconds: w.end,
+    }));
+  }
+
+  // Split original text into display units
+  const isChinese = /[一-鿿]/.test(originalText);
+  const units = isChinese
+    ? originalText.split("").filter((c) => c.trim())
+    : originalText.split(/\s+/).filter((w) => w);
+
+  const totalOrigChars = units.reduce((sum, u) => sum + u.length, 0);
+
+  let charOffset = 0;
+  return units.map((unit) => {
+    const startRatio = charOffset / totalOrigChars;
+    const endRatio = (charOffset + unit.length) / totalOrigChars;
+
+    const startIdx = Math.min(
+      Math.floor(startRatio * charTimes.length),
+      charTimes.length - 1
+    );
+    const endIdx = Math.min(
+      Math.floor(endRatio * charTimes.length),
+      charTimes.length - 1
+    );
+
+    const startSeconds = charTimes[startIdx];
+    const endSeconds = charTimes[Math.max(endIdx, startIdx)];
+
+    charOffset += unit.length;
+
+    return { word: unit, startSeconds, endSeconds };
+  });
+}