feat(transcripts): stage corrected .cleaned.txt over raw ASR

bin/stage_completed_transcripts.rb

@@ -2,9 +2,19 @@
 require 'yaml'
 require 'fileutils'
 
-assets = YAML.load_file("_data/video_assets.yml")["items"]
-staging_dir = "tmp/transcript-id-staging"
-FileUtils.mkdir_p(staging_dir)
+# Choose one transcript file per video id, preferring the known-terms-corrected
+# `<id>.cleaned.txt` (the /transcriptions learning-loop output) over the raw
+# `<id>.txt` / `<id>.stitched.txt`, so operator corrections are what reach the
+# site. Video ids carry no dots (YouTube [A-Za-z0-9_-]{11}, Vimeo digits), so the
+# id is the basename up to the first dot — which also folds a source's
+# `.stitched` / `.cleaned` variants together.
+def preferred_transcript_files(paths)
+  paths.group_by { |p| File.basename(p).split('.').first }.map do |_video_id, group|
+    group.find { |p| p.end_with?('.cleaned.txt') } ||
+      group.find { |p| p.end_with?('.stitched.txt') } ||
+      group.min_by { |p| File.basename(p).length }
+  end
+end
 
 # Diarization sidecars (`diarization.json`) may live next to the `.txt` in
 # either the recipe/short path (~/Downloads/transcripts/<id>/) or the ingest
@@ -23,36 +33,41 @@ def stage_sidecar(txt_file, staging_key, staging_dir)
   true
 end
 
-staged_count = 0
-staged_diarization = 0
-transcript_globs = [
-  File.expand_path("~/Downloads/transcripts/*/*.txt"),
-  File.expand_path("~/.local/state/zdots/ingest-sources/**/*.txt")
-]
+if __FILE__ == $PROGRAM_NAME
+  assets = YAML.load_file("_data/video_assets.yml")["items"]
+  staging_dir = "tmp/transcript-id-staging"
+  FileUtils.mkdir_p(staging_dir)
 
-Dir.glob(transcript_globs).each do |txt_file|
-  video_id = File.basename(txt_file, ".txt")
-
-  # Find the corresponding video_asset_id (check both YouTube and Vimeo)
-  asset = assets.find do |a|
-    a["platforms"]&.any? { |p| (p["platform"] == "youtube" || p["platform"] == "vimeo") && p["asset_id"] == video_id }
-  end
+  staged_count = 0
+  staged_diarization = 0
+  transcript_globs = [
+    File.expand_path("~/Downloads/transcripts/*/*.txt"),
+    File.expand_path("~/.local/state/zdots/ingest-sources/**/*.txt")
+  ]
 
-  if asset
-    video_asset_id = asset["id"]
-    dest_txt = File.join(staging_dir, "#{video_asset_id}.txt")
+  preferred_transcript_files(Dir.glob(transcript_globs)).each do |txt_file|
+    video_id = File.basename(txt_file).split('.').first
 
-    unless File.exist?(dest_txt)
-      FileUtils.cp(txt_file, dest_txt)
-      puts "Staged: #{video_id} -> #{video_asset_id}.txt"
-      staged_count += 1
+    # Find the corresponding video_asset_id (check both YouTube and Vimeo)
+    asset = assets.find do |a|
+      a["platforms"]&.any? { |p| (p["platform"] == "youtube" || p["platform"] == "vimeo") && p["asset_id"] == video_id }
     end
 
-    staged_diarization += 1 if stage_sidecar(txt_file, video_asset_id, staging_dir)
-  else
-    # Fallback: check if the filename itself is a video_asset_id
-    asset_by_id = assets.find { |a| a["id"] == video_id }
-    if asset_by_id
+    if asset
+      video_asset_id = asset["id"]
+      dest_txt = File.join(staging_dir, "#{video_asset_id}.txt")
+
+      unless File.exist?(dest_txt)
+        FileUtils.cp(txt_file, dest_txt)
+        puts "Staged: #{video_id} -> #{video_asset_id}.txt"
+        staged_count += 1
+      end
+
+      staged_diarization += 1 if stage_sidecar(txt_file, video_asset_id, staging_dir)
+    else
+      # Fallback: check if the filename itself is a video_asset_id
+      asset_by_id = assets.find { |a| a["id"] == video_id }
+      if asset_by_id
         dest_txt = File.join(staging_dir, "#{video_id}.txt")
         unless File.exist?(dest_txt)
           FileUtils.cp(txt_file, dest_txt)
@@ -61,9 +76,10 @@ def stage_sidecar(txt_file, staging_key, staging_dir)
         end
 
         staged_diarization += 1 if stage_sidecar(txt_file, video_id, staging_dir)
+      end
     end
   end
-end
 
-puts "Staged #{staged_count} new transcripts."
-puts "Staged #{staged_diarization} diarization sidecars." if staged_diarization.positive?
+  puts "Staged #{staged_count} new transcripts."
+  puts "Staged #{staged_diarization} diarization sidecars." if staged_diarization.positive?
+end

spec/bin/stage_completed_transcripts_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "../../bin/stage_completed_transcripts"
+
+RSpec.describe "preferred_transcript_files" do
+  it "prefers the corrected .cleaned.txt over raw variants for a video id" do
+    files = %w[
+      /r/ABC123.txt
+      /r/ABC123.stitched.txt
+      /r/ABC123.stitched.cleaned.txt
+    ]
+    expect(preferred_transcript_files(files)).to eq(["/r/ABC123.stitched.cleaned.txt"])
+  end
+
+  it "falls back to .stitched.txt when no cleaned variant exists" do
+    files = %w[/r/ABC123.txt /r/ABC123.stitched.txt]
+    expect(preferred_transcript_files(files)).to eq(["/r/ABC123.stitched.txt"])
+  end
+
+  it "keeps the plain <id>.txt when it is the only variant" do
+    expect(preferred_transcript_files(["/d/ABC123.txt"])).to eq(["/d/ABC123.txt"])
+  end
+
+  it "returns exactly one file per distinct video id" do
+    files = %w[/r/ABC.txt /r/ABC.cleaned.txt /r/XYZ.txt]
+    expect(preferred_transcript_files(files).sort).to eq(["/r/ABC.cleaned.txt", "/r/XYZ.txt"])
+  end
+end