diff --git a/packages/tools/video-grabber/tests/test_transcript_summarize.py b/packages/tools/video-grabber/tests/test_transcript_summarize.py index 766d4ff0..d69f140e 100644 --- a/packages/tools/video-grabber/tests/test_transcript_summarize.py +++ b/packages/tools/video-grabber/tests/test_transcript_summarize.py @@ -236,3 +236,16 @@ def test_mixed_tv_and_radio_keys_sort_without_raising(): ]) keys = sorted(buckets, key=lambda k: tuple("" if p is None else str(p) for p in k)) assert len(keys) == 2 + + +def test_a_bare_apostrophe_is_not_treated_as_an_invented_word(): + # Found verifying 1,500 production rows: one summary was flagged as + # containing a word absent from its source, and the "word" was a lone + # apostrophe. It strips to empty, so it could never match anything. + src = "the smoke is still billowing out behind you as the events unfolded" + assert validate_abstract("the smoke ' is still billowing out behind you", src) is None + + +def test_contractions_and_possessives_still_tokenize(): + got = content_words("the city's tower isn't standing") + assert _stem("city") in got and _stem("tower") in got diff --git a/packages/tools/video-grabber/video_grabber/transcript/summarize.py b/packages/tools/video-grabber/video_grabber/transcript/summarize.py index 287f341e..13767f8e 100644 --- a/packages/tools/video-grabber/video_grabber/transcript/summarize.py +++ b/packages/tools/video-grabber/video_grabber/transcript/summarize.py @@ -67,7 +67,12 @@ haven't hasn't couldn't shouldn't wouldn't i'm we're they're it's that's there's """.split()) -_WORD = re.compile(r"[a-z0-9']+") +# Must START with an alphanumeric. "[a-z0-9']+" also matched a BARE apostrophe as +# a token; it strips to the empty string, can never match anything in the source, +# and so was reported as an invented word. That produced a spurious containment +# failure on real data -- harmless for tier 1, but in tier 2 every false rejection +# costs a real summary and falls back. +_WORD = re.compile(r"[a-z0-9]+(?:'[a-z0-9]+)*") @dataclass(frozen=True) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83745e6c..9f5d0b47 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3075,8 +3075,8 @@ packages: react: '>= 16.8.0 || ^18.0.0' react-dom: '>= 16.8.0 || ^18.0.0' - react-hook-form@7.83.0: - resolution: {integrity: sha512-AXt8cMCmx5a7u4uvpb2uRFVrWQhllI4pV+LSykxIac/hjt44TnQkmX9BKuQi2i+LDC62esmiLpilkav+kjVf/A==} + react-hook-form@7.84.0: + resolution: {integrity: sha512-+hWvQP6GLco56mDwrbU4XnHix8t1z90ltZsDIrREl+jnQFQxYLX8oAzqe/Xn8nHpmoXTY5M6oEXrAhbP1qevNQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^16.8.0 || ^17 || ^18 || ^19 @@ -4760,7 +4760,7 @@ snapshots: micromark-util-symbol: 2.0.1 react: 19.2.7 react-dom: 19.2.7(react@19.2.7) - react-hook-form: 7.83.0(react@19.2.7) + react-hook-form: 7.84.0(react@19.2.7) unidiff: 1.0.4 transitivePeerDependencies: - '@codemirror/language' @@ -7472,7 +7472,7 @@ snapshots: react: 19.2.7 react-dom: 19.2.7(react@19.2.7) - react-hook-form@7.83.0(react@19.2.7): + react-hook-form@7.84.0(react@19.2.7): dependencies: react: 19.2.7