From ee7ab63f217a4069f1f4225b4d566073f1383edb Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sat, 1 Aug 2026 03:02:12 +0000 Subject: [PATCH] fix(video-grabber): a bare apostrophe is not an invented word Found while verifying 1,500 production summaries against their own source transcripts: exactly one was reported as containing a word absent from its source, and the "word" was a lone apostrophe. _WORD was "[a-z0-9']+", which matches a standalone "'" as a token. _variants strips it to the empty string, so it can never match anything in the source and is always reported as invented. Requiring a token to START with an alphanumeric keeps contractions and possessives ("don't", "city's") while dropping the bare punctuation. Harmless in tier 1, where the line is rebuilt from source indices and the validator is only an assertion. It matters in tier 2, where every false rejection discards a real summary and falls back. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_transcript_summarize.py | 13 +++++++++++++ .../video_grabber/transcript/summarize.py | 7 ++++++- pnpm-lock.yaml | 8 ++++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/tools/video-grabber/tests/test_transcript_summarize.py b/packages/tools/video-grabber/tests/test_transcript_summarize.py index 766d4ff0..d69f140e 100644 --- a/packages/tools/video-grabber/tests/test_transcript_summarize.py +++ b/packages/tools/video-grabber/tests/test_transcript_summarize.py @@ -236,3 +236,16 @@ def test_mixed_tv_and_radio_keys_sort_without_raising(): ]) keys = sorted(buckets, key=lambda k: tuple("" if p is None else str(p) for p in k)) assert len(keys) == 2 + + +def test_a_bare_apostrophe_is_not_treated_as_an_invented_word(): + # Found verifying 1,500 production rows: one summary was flagged as + # containing a word absent from its source, and the "word" was a lone + # apostrophe. It strips to empty, so it could never match anything. + src = "the smoke is still billowing out behind you as the events unfolded" + assert validate_abstract("the smoke ' is still billowing out behind you", src) is None + + +def test_contractions_and_possessives_still_tokenize(): + got = content_words("the city's tower isn't standing") + assert _stem("city") in got and _stem("tower") in got diff --git a/packages/tools/video-grabber/video_grabber/transcript/summarize.py b/packages/tools/video-grabber/video_grabber/transcript/summarize.py index 287f341e..13767f8e 100644 --- a/packages/tools/video-grabber/video_grabber/transcript/summarize.py +++ b/packages/tools/video-grabber/video_grabber/transcript/summarize.py @@ -67,7 +67,12 @@ haven't hasn't couldn't shouldn't wouldn't i'm we're they're it's that's there's """.split()) -_WORD = re.compile(r"[a-z0-9']+") +# Must START with an alphanumeric. "[a-z0-9']+" also matched a BARE apostrophe as +# a token; it strips to the empty string, can never match anything in the source, +# and so was reported as an invented word. That produced a spurious containment +# failure on real data -- harmless for tier 1, but in tier 2 every false rejection +# costs a real summary and falls back. +_WORD = re.compile(r"[a-z0-9]+(?:'[a-z0-9]+)*") @dataclass(frozen=True) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83745e6c..9f5d0b47 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3075,8 +3075,8 @@ packages: react: '>= 16.8.0 || ^18.0.0' react-dom: '>= 16.8.0 || ^18.0.0' - react-hook-form@7.83.0: - resolution: {integrity: sha512-AXt8cMCmx5a7u4uvpb2uRFVrWQhllI4pV+LSykxIac/hjt44TnQkmX9BKuQi2i+LDC62esmiLpilkav+kjVf/A==} + react-hook-form@7.84.0: + resolution: {integrity: sha512-+hWvQP6GLco56mDwrbU4XnHix8t1z90ltZsDIrREl+jnQFQxYLX8oAzqe/Xn8nHpmoXTY5M6oEXrAhbP1qevNQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^16.8.0 || ^17 || ^18 || ^19 @@ -4760,7 +4760,7 @@ snapshots: micromark-util-symbol: 2.0.1 react: 19.2.7 react-dom: 19.2.7(react@19.2.7) - react-hook-form: 7.83.0(react@19.2.7) + react-hook-form: 7.84.0(react@19.2.7) unidiff: 1.0.4 transitivePeerDependencies: - '@codemirror/language' @@ -7472,7 +7472,7 @@ snapshots: react: 19.2.7 react-dom: 19.2.7(react@19.2.7) - react-hook-form@7.83.0(react@19.2.7): + react-hook-form@7.84.0(react@19.2.7): dependencies: react: 19.2.7