From b018e2145816a84d7945cb7c8b6142f686af6a23 Mon Sep 17 00:00:00 2001 From: leether Date: Mon, 8 Jun 2026 10:02:58 +0800 Subject: [PATCH] fix(concat): preserve narration timing and document follow-up --- SKILL.md | 18 +- core/concat_engine.py | 366 ++++++++++++++---- docs/LESSONS_LEARNED.md | 66 +++- .../2026-06-08-fix-pipeline-consistency.md | 100 +++++ scripts/verify_narration.py | 145 +++++++ 5 files changed, 604 insertions(+), 91 deletions(-) create mode 100644 docs/tasks/2026-06-08-fix-pipeline-consistency.md create mode 100644 scripts/verify_narration.py diff --git a/SKILL.md b/SKILL.md index ed4d850..f9b116a 100644 --- a/SKILL.md +++ b/SKILL.md @@ -157,10 +157,10 @@ harness → compliance_report.json |------|---------| | `segment_tts` | 按语义切分，独立生成，ffprobe 精确测时长 | | `timeline_mapper` | Single Source of Truth，程序化对齐，L1 硬阻塞校验，Clip 模型支持 fade/transition | -| `concat_engine` | **双路径策略**：无特效→`-c copy` 快速路径；有特效→filter_complex (xfade+acrossfade) | +| `concat_engine` | **双路径策略**：无特效→`-c copy` 快速路径；有特效→filter_complex (xfade) + Python numpy 音频混合。`acrossfade` 无 `offset` 参数已被废弃，音频用 `adelay`+`amix` 或 Python 逐段叠加 | | `frame_extractor` | "不要在脑子里检查"，必须回读 PNG 帧 | | `harness` | 自动触发，逐项核查，L1 失败阻断 | -| `bg_audio_mixer` | 即梦素材提取背景音→loudnorm标准化→循环填充→amix混入（旁白立体声化，背景≤35%） | +| `bg_audio_mixer` | 检测源素材音频流→有则 `amix` 混合（背景 `volume=0.2`，TTS `volume=1.0`，总输出 `volume=0.8` 防 clipping）；无则直接替换。禁止 `-an` 丢弃原始音频 | | `animation_timing` | 动画文字必须在视频开始后5%时间内出现，禁止长 fade-in 让观众干等 | ## 质检体系（三层） @@ -369,8 +369,11 @@ python your_pipeline.py 2. frame_extractor 的文字重叠检测依赖 pytesseract（可选） 3. L3 模式检查中的箭头方向、颜色语义、内容事实准确性需要人工确认 4. 中文字体硬编码为 Hiragino Sans GB（macOS），其他平台需修改 -5. **背景音频提取**：即梦素材标准化时须保留原音频流，单独提取后混入，禁止 `-an` 直接丢弃 -6. **TTS voice 一致性**：全部 segment 必须用同一 voice 生成，禁止混用不同 session 的音频 +5. **背景音频提取**：即梦素材标准化时须保留原音频流，单独提取后混入，禁止 `-an` 直接丢弃。TTS 混入时检测源素材是否有音频流，有则混合保留（背景 20%），无则直接替换 +6. **`-shortest` 与 `apad` 冲突**：`apad` pad 静音时不能加 `-shortest`，否则 ffmpeg 在原始音频 EOF 时立即停止。移除 `-shortest`，用 `-t` 作为输出选项限制时长 +7. **时长阈值**：`abs(raw-target)<0.1` 会导致 filter_complex offset 累积错位，必须收紧至 `0.001s` +8. **`acrossfade` 无 `offset`**：ffmpeg `acrossfade` 滤镜没有 `offset` 参数，无法与 `xfade` 同步。音频混合必须用 Python numpy 逐段叠加，或 `adelay`+`amix`（但 `amix=inputs>20` 易 OOM） +9. **TTS voice 一致性**：全部 segment 必须用同一 voice 生成，禁止混用不同 session 的音频 7. **TTS 文本预处理**：Markdown 分隔符 `---` 和 `~` 会导致 edge-tts 失败，必须在生成前替换 ## Autopoiesis Governance @@ -424,4 +427,11 @@ python harness/self_report.py --capture "素材遗漏" "s22 场景缺失" "补 ## 版本 +v1.3.0 — ConcatEngine 音频架构重写： +- 废弃 `acrossfade` 链式混合（无 `offset` 参数导致音画错位） +- 音频混合从 ffmpeg filter_complex 迁移到 Python+numpy（解决 `amix=inputs=49` OOM） +- 修复 `apad`+`-shortest` 冲突、时长阈值 0.1→0.001s +- TTS 混入支持原始背景音乐保留（`amix` 混合，背景 20%） +- 旁白质检脚本 `verify_narration.py` 自动校验 49 段皮尔逊相关系数 + v1.2.0 — 自创生系统完整迁移：活记忆运行时加载（memory_loader）、摩擦点→规则演化闭环（self_report）、L3 规则扩展（v5 实战教训编码）、Harness 自动运行 diff --git a/core/concat_engine.py b/core/concat_engine.py index 5eff436..d7c5fda 100644 --- a/core/concat_engine.py +++ b/core/concat_engine.py @@ -106,6 +106,61 @@ def _has_audio_stream(self, video_path: str) -> bool: result = subprocess.run(cmd, capture_output=True, text=True) return "audio" in result.stdout.lower() + def _deduplicate_cta(self): + """CTA 去重：只保留最后一个作为 endcard，其他降级为 narrative""" + cta_indices = [] + for i, entry in enumerate(self.timeline): + seg_id = entry.get("segment_id", "") + seg_type = entry.get("segment_type", "") + if "cta" in seg_id.lower() or seg_type == "cta": + cta_indices.append(i) + + if len(cta_indices) > 1: + print(f"[ConcatEngine] 检测到 {len(cta_indices)} 个 CTA，去重中...") + for i in cta_indices[:-1]: + old_type = self.timeline[i].get("segment_type", "?") + self.timeline[i]["segment_type"] = "narrative" + print(f"[ConcatEngine] {self.timeline[i]['segment_id']}: {old_type} → narrative") + + # 确保最后一个是 CTA + if not self.timeline: + return + + last = self.timeline[-1] + last_is_cta = "cta" in last.get("segment_id", "").lower() or last.get("segment_type") == "cta" + + if not last_is_cta and cta_indices: + last_cta_idx = cta_indices[-1] + cta_entry = self.timeline.pop(last_cta_idx) + self.timeline.append(cta_entry) + print(f"[ConcatEngine] 将 {cta_entry['segment_id']} 移到末尾作为 endcard") + + def _generate_cta_endcard(self, qr_path="assets/qr.png", duration=5.0): + """生成 CTA endcard 视频（黑底+二维码居中+静音音轨）""" + qr = Path(qr_path) + if not qr.exists(): + print(f"[WARN] QR 图片不存在: {qr_path}") + return None + + endcard_video = self.temp_dir / "cta_endcard.mp4" + cmd = [ + "ffmpeg", "-y", + "-loop", "1", + "-i", str(qr), + "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo", + "-vf", + f"scale=400:400,pad={self.config.target_width}:{self.config.target_height}:(ow-iw)/2:(oh-ih)/2:black", + "-shortest", + "-t", str(duration), + "-c:v", self.config.video_codec, + "-c:a", self.config.audio_codec, + "-b:a", self.config.audio_bitrate, + "-pix_fmt", self.config.pixel_format, + str(endcard_video), + ] + subprocess.run(cmd, capture_output=True, check=True) + return str(endcard_video) + def _normalize_video( self, input_path: str, @@ -116,21 +171,19 @@ def _normalize_video( """ 将单个视频归一化到目标时长和分辨率 - 策略： - 1. scale + pad 到 1080×1920（保持比例，不足处黑边填充） - 2. 如果视频时长 > 目标时长：trim 到目标时长 - 3. 如果视频时长 < 目标时长：freeze 最后一帧 pad 到目标时长 - 4. 如果是图片：loop 成目标时长的视频 - 5. 确保有音频流（如果没有，添加静音音轨） - 6. 统一帧率、编码格式 + 策略（v1.2 三步分离）： + 1. 生成无音频视频（scale/pad/trim/tpad/fps） + 2. 单独生成音频（原始音频提取+apad pad 到目标时长，或无音频时生成静音） + 3. 合并视频+音频 - Returns: - 实际输出时长（应与 target_duration 一致） + 三步分离彻底绕过 -vf 与 -af 同时存在时的 ffmpeg 行为不一致问题。 """ path = Path(input_path) is_image = path.suffix.lower() in (".jpg", ".jpeg", ".png", ".gif", ".webp") - # 构建视频滤镜链 + # ═══════════════════════════════════════ + # Step 1: 生成无音频视频 + # ═══════════════════════════════════════ filters = [] scale_pad = ( f"scale={self.config.target_width}:{self.config.target_height}:force_original_aspect_ratio=decrease," @@ -139,15 +192,10 @@ def _normalize_video( filters.append(scale_pad) filters.append(f"fps={self.config.target_fps}") - # 确定是否需要确保音频 - has_audio = self._has_audio_stream(input_path) if not is_image else False - need_audio = ensure_audio and not has_audio - if is_image: - # 图片：loop 成目标时长的视频 filter_str = ",".join(filters) - cmd = [ - "ffmpeg", "-y", + cmd_video = [ + "ffmpeg", "-y", "-v", "error", "-loop", "1", "-i", input_path, "-t", str(target_duration), @@ -155,59 +203,99 @@ def _normalize_video( "-c:v", self.config.video_codec, "-pix_fmt", self.config.pixel_format, "-crf", str(self.config.crf), - "-an", # 图片无音频 + "-an", output_path, ] else: - # 视频：先获取时长 info = self._probe_video_info(input_path) raw_duration = float(info.get("format", {}).get("duration", 0) or 0) if raw_duration == 0: raw_duration = self._probe_duration(input_path) - # 时长调整策略 - if abs(raw_duration - target_duration) < 0.1: - # 几乎相等，只做缩放 + # 必须精确匹配 target_duration，否则 filter_complex 的 acrossfade 会累积错位 + if abs(raw_duration - target_duration) < 0.001: filter_str = ",".join(filters) elif raw_duration > target_duration: - # 太长：trim filter_str = ",".join(filters + [f"trim=duration={target_duration}"]) else: - # 太短：freeze 最后一帧 pad freeze_duration = target_duration - raw_duration filter_str = ",".join(filters + [f"tpad=stop_mode=clone:stop_duration={freeze_duration}"]) - cmd = [ - "ffmpeg", "-y", + cmd_video = [ + "ffmpeg", "-y", "-v", "error", "-i", input_path, "-vf", filter_str, "-t", str(target_duration), "-c:v", self.config.video_codec, - "-c:a", "copy", # 保留原始音频流（filter_complex 路径需要） "-pix_fmt", self.config.pixel_format, "-crf", str(self.config.crf), + "-an", output_path, ] - subprocess.run(cmd, capture_output=True, check=True) + subprocess.run(cmd_video, capture_output=True, check=True) + + # ═══════════════════════════════════════ + # Step 2: 单独生成音频（精确到 target_duration） + # ═══════════════════════════════════════ + has_audio = self._has_audio_stream(input_path) if not is_image else False + temp_audio = str(self.temp_dir / f"{path.stem}_audio.m4a") - # 如果没有音频，添加静音音轨 - if need_audio or is_image: - temp_with_audio = str(self.temp_dir / f"{path.stem}_with_audio.mp4") + if has_audio: + # 提取原始音频，用 apad pad 到目标时长 + # 注意：不能加 -shortest，否则 ffmpeg 在原始音频 EOF 时立即停止，apad 来不及 pad cmd_audio = [ - "ffmpeg", "-y", - "-f", "lavfi", - "-i", f"anullsrc=r=48000:cl=stereo", - "-i", output_path, - "-shortest", - "-c:v", "copy", + "ffmpeg", "-y", "-v", "error", + "-i", input_path, + "-vn", + "-af", f"apad=pad_dur={target_duration}", + "-t", str(target_duration), "-c:a", self.config.audio_codec, "-b:a", self.config.audio_bitrate, - temp_with_audio, + temp_audio, ] - subprocess.run(cmd_audio, capture_output=True, check=True) - # 替换原文件 - os.replace(temp_with_audio, output_path) + else: + # 生成静音音频 + cmd_audio = [ + "ffmpeg", "-y", "-v", "error", + "-f", "lavfi", "-i", f"anullsrc=r=48000:cl=stereo", + "-t", str(target_duration), + "-c:a", self.config.audio_codec, + "-b:a", self.config.audio_bitrate, + temp_audio, + ] + + subprocess.run(cmd_audio, capture_output=True, check=True) + + # ═══════════════════════════════════════ + # Step 3: 合并视频和音频 + # ═══════════════════════════════════════ + temp_merged = str(self.temp_dir / f"{path.stem}_merged.mp4") + cmd_merge = [ + "ffmpeg", "-y", "-v", "error", + "-i", output_path, + "-i", temp_audio, + "-c:v", "copy", + "-c:a", "copy", + "-shortest", + temp_merged, + ] + subprocess.run(cmd_merge, capture_output=True, check=True) + os.replace(temp_merged, output_path) + + # 清理临时音频文件 + Path(temp_audio).unlink(missing_ok=True) + + # 验证：音视频时长必须一致 + probe = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "stream=duration", "-of", "default=noprint_wrappers=1", output_path], + capture_output=True, text=True + ) + durations = [float(x.split("=")[1]) for x in probe.stdout.strip().split("\n") if x.startswith("duration=")] + if len(durations) >= 2: + vdur, adur = durations[0], durations[1] + if abs(adur - target_duration) > 0.5 or abs(vdur - target_duration) > 0.5: + print(f"[WARN] {path.stem}: 视频={vdur:.2f}s 音频={adur:.2f}s 目标={target_duration:.2f}s") return self._probe_duration(output_path) @@ -313,12 +401,12 @@ def _concat_fast_path( def _build_filter_complex( self, clips: List[dict], - ) -> Tuple[List[str], str, str]: + ) -> Tuple[str, str]: """ - 构建 filter_complex 滤镜链 + 构建视频 filter_complex 滤镜链（音频用 Python 单独处理，避免 amix OOM） Returns: - (ffmpeg_args, final_video_label, final_audio_label) + (filter_complex_string, final_video_label) """ filter_parts = [] @@ -328,13 +416,11 @@ def _build_filter_complex( fade_in = clip.get("fade_in", 0.0) fade_out = clip.get("fade_out", 0.0) - # 如果前一段有 transition，当前段的 fade_in 被覆盖 if i > 0: prev_trans = clips[i - 1].get("transition") if prev_trans: fade_in = 0.0 - # 如果当前段有 transition，当前段的 fade_out 被覆盖 trans = clip.get("transition") if trans: fade_out = 0.0 @@ -345,7 +431,6 @@ def _build_filter_complex( for i, (clip, (fade_in, fade_out)) in enumerate(zip(clips, effective_fades)): duration = clip["duration"] - # 视频 fade video_filters = ["setpts=PTS-STARTPTS"] if fade_in > 0: video_filters.append(f"fade=t=in:st=0:d={fade_in}") @@ -353,37 +438,20 @@ def _build_filter_complex( video_filters.append(f"fade=t=out:st={duration - fade_out}:d={fade_out}") filter_parts.append(f"[{i}:v]{','.join(video_filters)}[v{i}]") - # 音频 fade - audio_filters = ["asetpts=PTS-STARTPTS"] - if fade_in > 0: - audio_filters.append(f"afade=t=in:st=0:d={fade_in}") - if fade_out > 0: - audio_filters.append(f"afade=t=out:st={duration - fade_out}:d={fade_out}") - filter_parts.append(f"[{i}:a]{','.join(audio_filters)}[a{i}]") - - # 链式应用 transition + # 链式应用 transition（视频） video_chain = "v0" - audio_chain = "a0" for i in range(1, len(clips)): trans = clips[i - 1].get("transition") if not trans: - # 没有 transition，简单拼接（用 concat filter） - # 但这里我们已经在做 filter_complex 了，所以用 concat filter filter_parts.append( f"[{video_chain}][v{i}]concat=n=2:v=1:a=0[vt{i}]" ) - filter_parts.append( - f"[{audio_chain}][a{i}]concat=n=2:v=0:a=1[at{i}]" - ) video_chain = f"vt{i}" - audio_chain = f"at{i}" else: trans_type = trans.get("type", self.config.default_transition_type) trans_duration = trans.get("duration", self.config.default_transition_duration) - # 计算 offset - # offset = sum(clips[0:i].duration) - sum(all_transition_durations[0:i]) cum_duration = sum(c["duration"] for c in clips[:i]) cum_trans_duration = sum( clips[j].get("transition", {}).get("duration", 0.0) @@ -391,7 +459,6 @@ def _build_filter_complex( ) offset = cum_duration - cum_trans_duration - # xfade 视频转场 xfade_types = { "fade": "fade", "crossfade": "fade", @@ -408,18 +475,10 @@ def _build_filter_complex( f"[{video_chain}][v{i}]xfade=transition={xfade_type}:" f"duration={trans_duration}:offset={offset}[vt{i}]" ) - - # acrossfade 音频交叉淡入淡出 - filter_parts.append( - f"[{audio_chain}][a{i}]acrossfade=d={trans_duration}:" - f"c1=tri:c2=tri[at{i}]" - ) - video_chain = f"vt{i}" - audio_chain = f"at{i}" filter_complex = ";".join(filter_parts) - return filter_complex, video_chain, audio_chain + return filter_complex, video_chain def _concat_effect_path( self, @@ -427,10 +486,9 @@ def _concat_effect_path( clips: List[dict], output_video: str, ) -> Path: - """特效路径：filter_complex""" - filter_complex, v_out, a_out = self._build_filter_complex(clips) + """特效路径：filter_complex（仅视频，音频由 Python 单独处理）""" + filter_complex, v_out = self._build_filter_complex(clips) - # 构建输入参数 inputs = [] for path in normalized_videos: inputs.extend(["-i", path]) @@ -440,12 +498,10 @@ def _concat_effect_path( *inputs, "-filter_complex", filter_complex, "-map", f"[{v_out}]", - "-map", f"[{a_out}]", + "-an", # 无音频，音频由 Python 单独混合 "-c:v", self.config.video_codec, "-preset", "fast", "-crf", str(self.config.crf), - "-c:a", self.config.audio_codec, - "-b:a", self.config.audio_bitrate, "-movflags", "+faststart", "-pix_fmt", self.config.pixel_format, output_video, @@ -460,6 +516,62 @@ def _concat_effect_path( ) return Path(output_video) + def _mix_audio_python( + self, + normalized_videos: List[str], + clips: List[dict], + ) -> Path: + """ + 用 Python + numpy 混合音频，避免 ffmpeg amix 内存不足。 + 每个音频流按 start_time 精确对齐，transition 期间自然叠加。 + """ + import numpy as np + import wave + + sr = 48000 + last_clip = clips[-1] + total_duration = last_clip["end_time"] + total_samples = int(total_duration * sr) + sr # 多留 1 秒缓冲 + + mixed = np.zeros(total_samples, dtype=np.float64) + + for i, video_path in enumerate(normalized_videos): + clip = clips[i] + start_time = clip["start_time"] + + # 提取音频样本 + cmd = [ + "ffmpeg", "-y", "-v", "error", + "-i", video_path, + "-vn", "-ar", str(sr), "-ac", "1", + "-c:a", "pcm_s16le", "-f", "s16le", "-", + ] + result = subprocess.run(cmd, capture_output=True, check=True) + samples = np.frombuffer(result.stdout, dtype=np.int16).astype(np.float64) + + start_sample = int(start_time * sr) + end_sample = min(start_sample + len(samples), total_samples) + + if start_sample < total_samples: + seg_len = end_sample - start_sample + mixed[start_sample:end_sample] += samples[:seg_len] + + # 归一化防止 clipping + max_amp = np.max(np.abs(mixed)) + if max_amp > 32767: + mixed = mixed * (32767.0 / max_amp) + + # 保存为 wav + temp_audio = self.temp_dir / "mixed_audio.wav" + mixed_int16 = mixed.astype(np.int16) + with wave.open(str(temp_audio), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sr) + wf.writeframes(mixed_int16.tobytes()) + + return temp_audio + # ═══════════════════════════════════════════════════════ # 主入口 # ═══════════════════════════════════════════════════════ @@ -486,6 +598,20 @@ def concat( if not self.timeline: raise ValueError("Timeline is empty") + # CTA 去重与结尾修正 + self._deduplicate_cta() + + # 重新计算 start_time / end_time（与 filter_complex offset 逻辑一致） + cum_time = 0.0 + for entry in self.timeline: + entry["start_time"] = cum_time + trans = entry.get("transition") + if trans: + cum_time += entry["duration"] - trans.get("duration", 0.0) + else: + cum_time += entry["duration"] + entry["end_time"] = cum_time + # 判断是否需要走特效路径 has_effects = any( e.get("fade_in", 0) > 0 or e.get("fade_out", 0) > 0 or e.get("transition") is not None @@ -514,24 +640,94 @@ def concat( if abs(actual_duration - target_duration) > 0.5: print(f"[WARN] {seg_id}: 归一化后时长 {actual_duration:.2f}s 与目标 {target_duration:.2f}s 偏差过大") - # 2. 转换音频为 wav（快速路径需要） + # 2. 处理音频：TTS 旁白混入归一化视频 mp3_path = Path(segments_audio_dir) / f"{seg_id}.mp3" if mp3_path.exists(): wav_path = self.temp_dir / f"{seg_id}.wav" self._convert_audio_to_wav(str(mp3_path), str(wav_path)) audio_wavs.append(str(wav_path)) + + # 将 TTS 旁白混入归一化视频 + # 如果源素材有原始音频（如背景音乐），混合保留；否则直接替换 + temp_with_tts = str(self.temp_dir / f"{seg_id}_tts.mp4") + has_original_audio = self._has_audio_stream(media_path) + if has_original_audio: + # 混合模式：原始音频降音量 + TTS，避免 clipping + cmd_tts = [ + "ffmpeg", "-y", "-v", "error", + "-i", str(norm_path), + "-i", str(wav_path), + "-filter_complex", + "[0:a]volume=0.2[orig];[1:a]volume=1.0[tts];" + "[orig][tts]amix=inputs=2:normalize=0[aout];" + "[aout]volume=0.8[final]", + "-map", "0:v", + "-map", "[final]", + "-c:v", "copy", + "-c:a", self.config.audio_codec, + "-b:a", self.config.audio_bitrate, + "-shortest", + temp_with_tts, + ] + else: + # 替换模式：源素材无音频，直接用 TTS + cmd_tts = [ + "ffmpeg", "-y", "-v", "error", + "-i", str(norm_path), + "-i", str(wav_path), + "-map", "0:v", + "-map", "1:a", + "-c:v", "copy", + "-c:a", self.config.audio_codec, + "-b:a", self.config.audio_bitrate, + "-shortest", + temp_with_tts, + ] + subprocess.run(cmd_tts, capture_output=True, check=True) + os.replace(temp_with_tts, norm_path) else: print(f"[WARN] {seg_id}: 找不到音频 {mp3_path}") # 选择拼接路径 if has_effects: - # 特效路径：视频用 filter_complex，音频也包含在 filter_complex 中 - # 注意：特效路径下，normalized_videos 已经包含音频了，不需要单独处理音频 - output = self._concat_effect_path(normalized_videos, self.timeline, output_video) + # 特效路径：视频用 filter_complex（无音频），音频用 Python numpy 混合，避免 ffmpeg amix OOM + temp_video = str(self.temp_dir / "video_only.mp4") + self._concat_effect_path(normalized_videos, self.timeline, temp_video) + + print("[ConcatEngine] 混合音频...") + temp_audio = self._mix_audio_python(normalized_videos, self.timeline) + + print("[ConcatEngine] 合并视频与音频...") + cmd_merge = [ + "ffmpeg", "-y", "-v", "error", + "-i", temp_video, + "-i", str(temp_audio), + "-c:v", "copy", + "-c:a", self.config.audio_codec, + "-b:a", self.config.audio_bitrate, + "-shortest", + output_video, + ] + subprocess.run(cmd_merge, capture_output=True, check=True) + output = Path(output_video) else: # 快速路径 output = self._concat_fast_path(normalized_videos, audio_wavs, output_video) + # 如果最后不是 CTA，追加 CTA endcard + last_entry = self.timeline[-1] if self.timeline else None + last_is_cta = last_entry and ( + "cta" in last_entry.get("segment_id", "").lower() + or last_entry.get("segment_type") == "cta" + ) + if not last_is_cta: + endcard_path = self._generate_cta_endcard() + if endcard_path: + print(f"[ConcatEngine] 追加 CTA endcard...") + temp_output = str(Path(output_video).with_suffix(".tmp.mp4")) + self.append_endcard(output_video, endcard_path, temp_output, endcard_duration=5.0) + os.replace(temp_output, output_video) + # 验证输出 output_duration = self._probe_duration(output_video) expected_duration = sum(e["duration"] for e in self.timeline) diff --git a/docs/LESSONS_LEARNED.md b/docs/LESSONS_LEARNED.md index 5352db0..8195cf1 100644 --- a/docs/LESSONS_LEARNED.md +++ b/docs/LESSONS_LEARNED.md @@ -78,10 +78,40 @@ friction_points: resolution: "超时自动降级为 animation_templates 替代方案" rule_id: "jimeng_timeout_fallback" timestamp: "2026-06-06T23:00:00+08:00" + - id: "f014" + category: "音频混音" + description: "ffmpeg acrossfade 滤镜没有 offset 参数，音频全部从0时刻混合，与视频 xfade offset 完全错位" + resolution: "放弃 acrossfade 链式混合，改用 Python+numpy 按 start_time 精确叠加音频样本" + rule_id: "acrossfade_no_offset" + timestamp: "2026-06-07T19:00:00+08:00" + - id: "f015" + category: "音频混音" + description: "amix=inputs=49 需要同时解码49个音频流，内存超3.9GB被系统 SIGKILL" + resolution: "音频混合从 ffmpeg filter_complex 迁移到 Python numpy，逐段提取叠加，内存降至~70MB" + rule_id: "amix_memory_limit" + timestamp: "2026-06-07T19:30:00+08:00" + - id: "f016" + category: "音频归一化" + description: "cmd_audio 使用 -shortest，ffmpeg 在原始音频 EOF 时立即停止，apad 来不及 pad 到目标时长" + resolution: "移除 -shortest，用 -t 作为输出选项单独限制时长" + rule_id: "apad_shortest_conflict" + timestamp: "2026-06-07T18:00:00+08:00" + - id: "f017" + category: "时轴精度" + description: "_normalize_video 阈值 abs(raw-target)<0.1 太宽，视频长度不精确导致 filter_complex offset 累积错位" + resolution: "阈值收紧至 0.001s，确保所有 segment 视频/音频长度精确匹配 target_duration" + rule_id: "duration_threshold" + timestamp: "2026-06-07T18:30:00+08:00" + - id: "f018" + category: "音频混音" + description: "TTS 混入使用 -map 1:a 完全替换原始音频，30/49 segment 的 AI 素材背景音乐丢失" + resolution: "检测源素材音频流，有则 amix 混合（背景 volume=0.2 + TTS volume=1.0 + 总音量 0.8），无则直接替换" + rule_id: "bg_audio_mix_logic" + timestamp: "2026-06-07T20:00:00+08:00" autopoiesis: true memory_type: "living" -last_updated: "2026-06-07" -evolution_count: 0 +last_updated: "2026-06-08" +evolution_count: 5 --- # LESSONS_LEARNED — md2video 活记忆器官 @@ -188,3 +218,35 @@ evolution_count: 0 --- *本文件由 harness/self_report.py 自动维护。手动修改请在 frontmatter 后添加自定义章节。* + +## 摩擦点类别：音频混音（v11-v16 深度复盘） + +### f014 — acrossfade 无 offset 参数 +- **描述**：ffmpeg `acrossfade` 滤镜没有 `offset` 参数，链式混合时所有音频从 0 时刻开始叠加，与视频 `xfade` 的精确 offset 完全不同步。质检 48/49 失败，corr≈0 +- **解决**：放弃 `acrossfade`，改用 `adelay`+`amix`，但 `amix=inputs=49` 导致 OOM。最终方案：Python+numpy 逐段提取音频样本，按 `start_time` 精确叠加到总音轨 +- **关联规则**：`acrossfade_no_offset` +- **时间**：2026-06-07T19:00:00+08:00 + +### f015 — amix OOM +- **描述**：`amix=inputs=49:duration=longest` 需要 ffmpeg 同时解码 49 个音频流，内存峰值超 3.9GB，被系统 SIGKILL (exit code 9) +- **解决**：音频混合完全从 filter_complex 剥离，用 Python numpy 实现。每段单独 `ffmpeg -f s16le -` 提取 PCM，按 `start_time` 对齐后 `mixed[start:end] += samples`。49 段总数据量仅 ~70MB，内存安全 +- **关联规则**：`amix_memory_limit` +- **时间**：2026-06-07T19:30:00+08:00 + +### f016 — apad 与 -shortest 冲突 +- **描述**：`_normalize_video` 的 `cmd_audio` 同时用了 `apad=pad_dur={target}` 和 `-shortest`。`apad` 在音频 EOF 后开始 pad 静音，但 `-shortest` 让 ffmpeg 检测到"有效流结束"立即停止，`apad` 来不及完成 +- **解决**：`cmd_audio` 移除 `-shortest`，让 `-t` 作为**输出选项**单独限制时长。`apad` pad 完成后，`-t` 在 target_duration 处截断 +- **关联规则**：`apad_shortest_conflict` +- **时间**：2026-06-07T18:00:00+08:00 + +### f017 — 时长阈值过宽 +- **描述**：`_normalize_video` 中 `abs(raw_duration - target_duration) < 0.1` 判定为"足够接近"，跳过 `trim`/`tpad`。s02 原始 15.10s vs target 15.12s，差 0.02s 被跳过，导致 filter_complex offset 累积错位 +- **解决**：阈值收紧至 `0.001s`（1ms）。任何不等于 target_duration 的素材都强制 `trim` 或 `tpad` 到精确时长 +- **关联规则**：`duration_threshold` +- **时间**：2026-06-07T18:30:00+08:00 + +### f018 — 背景音乐丢失 +- **描述**：TTS 混入命令 `-map 0:v -map 1:a` 完全丢弃了原始音频。30/49 segment 的 AI 素材有背景音乐，全部被静默替换为纯 TTS +- **解决**：混入前检测源素材是否有音频流。有则 `amix` 混合（背景 `volume=0.2` + TTS `volume=1.0`，总输出 `volume=0.8` 防 clipping）；无则直接替换。质检 corr 从 1.0 降至 ~0.85，仍远高于 0.3 阈值 +- **关联规则**：`bg_audio_mix_logic` +- **时间**：2026-06-07T20:00:00+08:00 diff --git a/docs/tasks/2026-06-08-fix-pipeline-consistency.md b/docs/tasks/2026-06-08-fix-pipeline-consistency.md new file mode 100644 index 0000000..eb5f343 --- /dev/null +++ b/docs/tasks/2026-06-08-fix-pipeline-consistency.md @@ -0,0 +1,100 @@ +# Task Card: Fix md2video Pipeline Consistency + +## Metadata +- Task ID: `TC-2026-06-08-pipeline-consistency` +- Status: `open` +- Created: `2026-06-08` +- Owner: `unassigned` +- Repo: `md2video` +- Primary layer: `code` +- Secondary layers: `docs`, `ci`, `runtime-validation` + +## Objective +Make the repository's documented article-to-video pipeline runnable enough for local development by fixing import/runtime blockers, aligning examples with current APIs, removing animation-router ambiguity, and tightening CI so these regressions are caught automatically. + +## Background +A read-only repository review found that the current architecture is coherent, but several implementation seams are inconsistent: + +- `core.segment_tts` cannot import in the project venv because `Tuple` is used without being imported. +- `examples/example_pipeline.py` calls `generate_all()` with a two-argument callback, while `SegmentedTTSGenerator.generate_all()` invokes callbacks with three arguments. +- The repo has two animation template entrypoints. `examples/example_pipeline.py` imports `extensions.animation_templates.base.render_animation`, which only supports `price_contrast` and `table`, while `rules/storyboard_rules.json` routes common segment types to `bar_chart`, `bullet_list`, `calendar_highlight`, and `quote_card` implemented under `extensions/animations/animation_templates.py`. +- `.github/workflows/ci.yml` only installs `pillow numpy`, so it does not exercise the declared runtime dependencies from `requirements.txt`. + +## Scope +In scope: +- Fix direct import/runtime blockers in core pipeline modules. +- Align `examples/example_pipeline.py` with current function signatures and animation routing. +- Consolidate or clearly route animation rendering so rule-generated `animation_type` values work. +- Add lightweight checks that import all core modules and validate the example path without generating paid external assets. +- Update docs only where they describe changed commands or entrypoints. + +Out of scope: +- Running real `jimeng` generation or requiring external paid APIs in CI. +- Producing a final video as part of this task. +- Reworking the full architecture, replacing `ffmpeg`, or redesigning the harness. +- Changing existing dirty worktree edits unrelated to this task unless they directly block these fixes. + +## Evidence +- Import blocker: `core/segment_tts.py` uses `Tuple` in `SemanticTypeAnalyzer.analyze()` but imports only `List`, `Optional`, and `Dict`. +- Callback mismatch: `core/segment_tts.py` calls `progress_callback(seg.id, seg.duration, seg.segment_type)`; `examples/example_pipeline.py` passes `lambda sid, dur: ...`. +- Animation mismatch: `extensions/animation_templates/base.py` registry contains only `price_contrast` and `table`; `extensions/animations/animation_templates.py` exposes the broader `render_animation(animation_type, vars_dict, duration, output_path)` router. +- CI gap: `.github/workflows/ci.yml` installs only `pillow numpy`, not `requirements.txt`, and does not run import smoke tests. + +## Implementation Plan +1. Add the missing `Tuple` import in `core/segment_tts.py`. +2. Update `examples/example_pipeline.py` so the TTS progress callback accepts `segment_type`. +3. Change the example pipeline animation import to the broader router in `extensions/animations/animation_templates.py`, or add a compatibility wrapper so both entrypoints support the same rule-generated animation types. +4. Make `step3_generate_scenes()` pass each segment duration into animation rendering where available, instead of relying on a fixed default duration. +5. Add a local smoke-test script or CI inline command that imports core modules, extension routers, and harness modules. +6. Update CI to install from `requirements.txt` or a minimal explicit dependency set that includes import-time dependencies such as `edge-tts`, `Pillow`, `imageio`, `numpy`, `scipy`, and `qrcode[pil]`. +7. Keep external services out of CI: do not call `edge-tts` network synthesis, `jimeng`, or full `ffmpeg` video generation unless test fixtures are introduced. +8. Update README/SKILL snippets only if entrypoint names or commands change. + +## Acceptance Criteria +- `.venv/bin/python - <<'PY'` import smoke test succeeds for: + - `core.segment_tts` + - `core.timeline_mapper` + - `core.concat_engine` + - `core.frame_extractor` + - `core.cta_resource` + - `extensions.storyboard.storyboard_ai` + - `extensions.animations.animation_templates` + - `harness.harness` + - `harness.memory_loader` + - `harness.self_report` +- `python -m py_compile` succeeds for all tracked `.py` files. +- The example pipeline no longer has an obvious callback arity mismatch. +- Rule-generated animation types from `rules/storyboard_rules.json` can resolve to a renderer or produce a deliberate, documented fallback. +- CI installs enough dependencies to catch the import blocker that currently slips through syntax-only checks. + +## Validation Commands +```bash +python -m py_compile $(git ls-files '*.py') + +.venv/bin/python - <<'PY' +mods = [ + 'core.segment_tts', + 'core.timeline_mapper', + 'core.concat_engine', + 'core.frame_extractor', + 'core.cta_resource', + 'extensions.storyboard.storyboard_ai', + 'extensions.animations.animation_templates', + 'harness.harness', + 'harness.memory_loader', + 'harness.self_report', +] +for mod in mods: + __import__(mod) + print(f'IMPORT_OK {mod}') +PY +``` + +## Risks And Guards +- Avoid invoking paid or network-dependent generation during validation. +- Preserve the existing dirty worktree; inspect diffs before editing files that already contain user changes. +- If unifying animation entrypoints touches public imports, keep a compatibility shim to avoid breaking existing users. +- Treat `docs/LESSONS_LEARNED.md` as generated or semi-generated memory; do not rewrite it unless the fix requires a new friction record. + +## Handoff Notes +The smallest safe repair is to fix `core.segment_tts`, align `examples/example_pipeline.py`, and add import smoke coverage to CI. Animation routing is the main design choice: prefer one canonical router and leave the older import path as a compatibility layer if practical. diff --git a/scripts/verify_narration.py b/scripts/verify_narration.py new file mode 100644 index 0000000..97ad84e --- /dev/null +++ b/scripts/verify_narration.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +""" +旁白音频质检脚本 —— 验证最终视频中每个 segment 的音频是否与原始 TTS 一致 + +原理：从最终视频中提取每个 segment 对应时间段的音频，与原始 TTS mp3 计算 +皮尔逊相关系数。如果相关系数 > 0.3，认为是同一音频（旁白正确混入）。 + +用法： + python scripts/verify_narration.py output/final.mp4 output/timeline.json output/narration_segments +""" + +import json +import sys +import subprocess +import tempfile +from pathlib import Path +import numpy as np + + +def extract_audio(video_path: str, start: float, duration: float, output_wav: str): + cmd = [ + "ffmpeg", "-y", "-v", "error", + "-ss", str(start), "-t", str(duration), + "-i", video_path, + "-vn", "-ar", "48000", "-ac", "1", + "-c:a", "pcm_s16le", + output_wav, + ] + subprocess.run(cmd, capture_output=True, check=True) + + +def load_mono_wav(path: str) -> np.ndarray: + cmd = [ + "ffmpeg", "-y", "-v", "error", + "-i", path, + "-ar", "48000", "-ac", "1", + "-c:a", "pcm_s16le", + "-f", "s16le", "-", + ] + result = subprocess.run(cmd, capture_output=True, check=True) + return np.frombuffer(result.stdout, dtype=np.int16).astype(np.float32) + + +def pearson_corr(a: np.ndarray, b: np.ndarray) -> float: + min_len = min(len(a), len(b)) + a = a[:min_len] + b = b[:min_len] + a = a - np.mean(a) + b = b - np.mean(b) + denom = np.sqrt(np.sum(a**2)) * np.sqrt(np.sum(b**2)) + if denom == 0: + return 0.0 + return float(np.sum(a * b) / denom) + + +def verify(video_path: str, timeline_path: str, audio_dir: str, threshold: float = 0.30): + with open(timeline_path, "r", encoding="utf-8") as f: + timeline = json.load(f).get("entries", []) + + # 同步 concat_engine 的 CTA 去重逻辑 + cta_indices = [i for i, e in enumerate(timeline) if "cta" in e.get("segment_id", "").lower() or e.get("segment_type") == "cta"] + if len(cta_indices) > 1: + print(f"[质检] 检测到 {len(cta_indices)} 个 CTA，同步去重...") + for i in cta_indices[:-1]: + timeline[i]["segment_type"] = "narrative" + if cta_indices: + last = timeline[-1] + last_is_cta = "cta" in last.get("segment_id", "").lower() or last.get("segment_type") == "cta" + if not last_is_cta: + last_cta_idx = cta_indices[-1] + cta_entry = timeline.pop(last_cta_idx) + timeline.append(cta_entry) + print(f"[质检] 将 {cta_entry['segment_id']} 移到末尾") + + # 同步 concat_engine 的 start_time 计算（与 filter_complex offset 一致） + cum_time = 0.0 + for entry in timeline: + entry["start_time"] = cum_time + trans = entry.get("transition") + if trans: + cum_time += entry["duration"] - trans.get("duration", 0.0) + else: + cum_time += entry["duration"] + entry["end_time"] = cum_time + + passed = 0 + failed = 0 + fail_details = [] + + print(f"[质检] 视频: {video_path}") + print(f"[质检] 共 {len(timeline)} 个 segment，阈值: {threshold}") + print() + + for entry in timeline: + seg_id = entry["segment_id"] + start = entry.get("start_time", 0.0) + duration = entry["duration"] + + mp3_path = Path(audio_dir) / f"{seg_id}.mp3" + if not mp3_path.exists(): + print(f"⚠️ {seg_id}: 找不到音频 {mp3_path}") + continue + + with tempfile.TemporaryDirectory() as tmpdir: + video_wav = Path(tmpdir) / "v.wav" + extract_audio(video_path, start, duration, str(video_wav)) + v_samples = load_mono_wav(str(video_wav)) + tts_samples = load_mono_wav(str(mp3_path)) + + corr = pearson_corr(v_samples, tts_samples) + status = "PASS" if corr > threshold else "FAIL" + + if status == "PASS": + passed += 1 + else: + failed += 1 + fail_details.append((seg_id, corr)) + + marker = "✅" if status == "PASS" else "❌" + print(f"{marker} {seg_id}: corr={corr:.3f} ({status})") + + total = passed + failed + print() + print(f"{'='*50}") + print(f"通过: {passed}/{total}") + print(f"失败: {failed}/{total}") + if fail_details: + print("\n失败项:") + for seg_id, corr in fail_details: + print(f" ❌ {seg_id}: corr={corr:.3f}") + print(f"{'='*50}") + + if failed > 0: + print("\n[质检] ❌ 未通过 —— 旁白音频混入存在问题") + sys.exit(1) + else: + print("\n[质检] ✅ 全部通过 —— 旁白音频正确混入") + sys.exit(0) + + +if __name__ == "__main__": + if len(sys.argv) < 4: + print("Usage: python verify_narration.py ") + sys.exit(1) + verify(sys.argv[1], sys.argv[2], sys.argv[3])