fix(video-grabber): a bare apostrophe is not an invented word

packages/tools/video-grabber/tests/test_transcript_summarize.py

@@ -236,3 +236,16 @@ def test_mixed_tv_and_radio_keys_sort_without_raising():
     ])
     keys = sorted(buckets, key=lambda k: tuple("" if p is None else str(p) for p in k))
     assert len(keys) == 2
+
+
+def test_a_bare_apostrophe_is_not_treated_as_an_invented_word():
+    # Found verifying 1,500 production rows: one summary was flagged as
+    # containing a word absent from its source, and the "word" was a lone
+    # apostrophe. It strips to empty, so it could never match anything.
+    src = "the smoke is still billowing out behind you as the events unfolded"
+    assert validate_abstract("the smoke ' is still billowing out behind you", src) is None
+
+
+def test_contractions_and_possessives_still_tokenize():
+    got = content_words("the city's tower isn't standing")
+    assert _stem("city") in got and _stem("tower") in got

packages/tools/video-grabber/video_grabber/transcript/summarize.py

@@ -67,7 +67,12 @@
 haven't hasn't couldn't shouldn't wouldn't i'm we're they're it's that's there's
 """.split())
 
-_WORD = re.compile(r"[a-z0-9']+")
+# Must START with an alphanumeric. "[a-z0-9']+" also matched a BARE apostrophe as
+# a token; it strips to the empty string, can never match anything in the source,
+# and so was reported as an invented word. That produced a spurious containment
+# failure on real data -- harmless for tier 1, but in tier 2 every false rejection
+# costs a real summary and falls back.
+_WORD = re.compile(r"[a-z0-9]+(?:'[a-z0-9]+)*")
 
 
 @dataclass(frozen=True)