diff --git a/bin/stage_completed_transcripts.rb b/bin/stage_completed_transcripts.rb index 5277c675..045a9942 100755 --- a/bin/stage_completed_transcripts.rb +++ b/bin/stage_completed_transcripts.rb @@ -2,9 +2,19 @@ require 'yaml' require 'fileutils' -assets = YAML.load_file("_data/video_assets.yml")["items"] -staging_dir = "tmp/transcript-id-staging" -FileUtils.mkdir_p(staging_dir) +# Choose one transcript file per video id, preferring the known-terms-corrected +# `.cleaned.txt` (the /transcriptions learning-loop output) over the raw +# `.txt` / `.stitched.txt`, so operator corrections are what reach the +# site. Video ids carry no dots (YouTube [A-Za-z0-9_-]{11}, Vimeo digits), so the +# id is the basename up to the first dot — which also folds a source's +# `.stitched` / `.cleaned` variants together. +def preferred_transcript_files(paths) + paths.group_by { |p| File.basename(p).split('.').first }.map do |_video_id, group| + group.find { |p| p.end_with?('.cleaned.txt') } || + group.find { |p| p.end_with?('.stitched.txt') } || + group.min_by { |p| File.basename(p).length } + end +end # Diarization sidecars (`diarization.json`) may live next to the `.txt` in # either the recipe/short path (~/Downloads/transcripts//) or the ingest @@ -23,36 +33,41 @@ def stage_sidecar(txt_file, staging_key, staging_dir) true end -staged_count = 0 -staged_diarization = 0 -transcript_globs = [ - File.expand_path("~/Downloads/transcripts/*/*.txt"), - File.expand_path("~/.local/state/zdots/ingest-sources/**/*.txt") -] +if __FILE__ == $PROGRAM_NAME + assets = YAML.load_file("_data/video_assets.yml")["items"] + staging_dir = "tmp/transcript-id-staging" + FileUtils.mkdir_p(staging_dir) -Dir.glob(transcript_globs).each do |txt_file| - video_id = File.basename(txt_file, ".txt") - - # Find the corresponding video_asset_id (check both YouTube and Vimeo) - asset = assets.find do |a| - a["platforms"]&.any? { |p| (p["platform"] == "youtube" || p["platform"] == "vimeo") && p["asset_id"] == video_id } - end + staged_count = 0 + staged_diarization = 0 + transcript_globs = [ + File.expand_path("~/Downloads/transcripts/*/*.txt"), + File.expand_path("~/.local/state/zdots/ingest-sources/**/*.txt") + ] - if asset - video_asset_id = asset["id"] - dest_txt = File.join(staging_dir, "#{video_asset_id}.txt") + preferred_transcript_files(Dir.glob(transcript_globs)).each do |txt_file| + video_id = File.basename(txt_file).split('.').first - unless File.exist?(dest_txt) - FileUtils.cp(txt_file, dest_txt) - puts "Staged: #{video_id} -> #{video_asset_id}.txt" - staged_count += 1 + # Find the corresponding video_asset_id (check both YouTube and Vimeo) + asset = assets.find do |a| + a["platforms"]&.any? { |p| (p["platform"] == "youtube" || p["platform"] == "vimeo") && p["asset_id"] == video_id } end - staged_diarization += 1 if stage_sidecar(txt_file, video_asset_id, staging_dir) - else - # Fallback: check if the filename itself is a video_asset_id - asset_by_id = assets.find { |a| a["id"] == video_id } - if asset_by_id + if asset + video_asset_id = asset["id"] + dest_txt = File.join(staging_dir, "#{video_asset_id}.txt") + + unless File.exist?(dest_txt) + FileUtils.cp(txt_file, dest_txt) + puts "Staged: #{video_id} -> #{video_asset_id}.txt" + staged_count += 1 + end + + staged_diarization += 1 if stage_sidecar(txt_file, video_asset_id, staging_dir) + else + # Fallback: check if the filename itself is a video_asset_id + asset_by_id = assets.find { |a| a["id"] == video_id } + if asset_by_id dest_txt = File.join(staging_dir, "#{video_id}.txt") unless File.exist?(dest_txt) FileUtils.cp(txt_file, dest_txt) @@ -61,9 +76,10 @@ def stage_sidecar(txt_file, staging_key, staging_dir) end staged_diarization += 1 if stage_sidecar(txt_file, video_id, staging_dir) + end end end -end -puts "Staged #{staged_count} new transcripts." -puts "Staged #{staged_diarization} diarization sidecars." if staged_diarization.positive? + puts "Staged #{staged_count} new transcripts." + puts "Staged #{staged_diarization} diarization sidecars." if staged_diarization.positive? +end diff --git a/spec/bin/stage_completed_transcripts_spec.rb b/spec/bin/stage_completed_transcripts_spec.rb new file mode 100644 index 00000000..c620e7a5 --- /dev/null +++ b/spec/bin/stage_completed_transcripts_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require_relative "../../bin/stage_completed_transcripts" + +RSpec.describe "preferred_transcript_files" do + it "prefers the corrected .cleaned.txt over raw variants for a video id" do + files = %w[ + /r/ABC123.txt + /r/ABC123.stitched.txt + /r/ABC123.stitched.cleaned.txt + ] + expect(preferred_transcript_files(files)).to eq(["/r/ABC123.stitched.cleaned.txt"]) + end + + it "falls back to .stitched.txt when no cleaned variant exists" do + files = %w[/r/ABC123.txt /r/ABC123.stitched.txt] + expect(preferred_transcript_files(files)).to eq(["/r/ABC123.stitched.txt"]) + end + + it "keeps the plain .txt when it is the only variant" do + expect(preferred_transcript_files(["/d/ABC123.txt"])).to eq(["/d/ABC123.txt"]) + end + + it "returns exactly one file per distinct video id" do + files = %w[/r/ABC.txt /r/ABC.cleaned.txt /r/XYZ.txt] + expect(preferred_transcript_files(files).sort).to eq(["/r/ABC.cleaned.txt", "/r/XYZ.txt"]) + end +end