diff --git a/README.md b/README.md index 37792d5..b588928 100644 --- a/README.md +++ b/README.md @@ -399,3 +399,18 @@ Set these as repository secrets (Settings > Secrets and variables > Actions): ### Running Trigger from the Actions tab (Run workflow, branch `main`). The bench accepts a `providers` input (default `albert,scaleway`); the judge accepts `mode` (`single`/`panel`) and `passes`. Each run consumes billable API calls, so the workflows are manual by design. Outputs are also uploaded as workflow artifacts. + +### Pushing local transcripts to the results dataset + +The CI only covers the API models. Local models (WhisperX, Kyutai, Cohere via MLX) run on your machine; push their outputs to the results dataset so the WER and judge cover every model: + +```bash +# Dry-run: show what would be pushed (and what is skipped) +uv run eval-transcript results push --dry-run + +# Push for real (single commit); optionally filter by file name +uv run eval-transcript results push +uv run eval-transcript results push --include whisperx +``` + +Safety: the command derives the public allowlist from the **corpus** dataset (`ground_truth/`) and uploads only transcripts whose sample is in it. Any local transcript for a sample absent from the public corpus (internal meeting, removed sample) is skipped, so private data never reaches the Hub. Override the repos with `--corpus` / `--results` (or `EVAL_CORPUS_REPO` / `EVAL_RESULTS_REPO`); pushing needs `HF_TOKEN` with write access. diff --git a/pyproject.toml b/pyproject.toml index fa481d9..eef7959 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.12" dependencies = [ "elevenlabs>=2.50.0", "httpx>=0.28.1", + "huggingface-hub>=0.24.0", "jiwer>=4.0.0", "python-dotenv>=1.2.2", "text2num>=2.4.0", diff --git a/src/eval_transcript/__init__.py b/src/eval_transcript/__init__.py index 923e636..4c5af9c 100644 --- a/src/eval_transcript/__init__.py +++ b/src/eval_transcript/__init__.py @@ -59,6 +59,13 @@ write_or_print_score_output, ) from eval_transcript.transcriptions import TranscriptionOutput, print_transcription_output, transcription_text +from eval_transcript.results import ( + DEFAULT_CORPUS_REPO, + DEFAULT_RESULTS_REPO, + DEFAULT_TRANSCRIPTIONS_DIR as RESULTS_DEFAULT_TRANSCRIPTIONS_DIR, + ResultsClient, + ResultsError, +) DEPRECATED_SOURCE_TRUTH_FLAG_MESSAGE = ( @@ -210,6 +217,20 @@ def main() -> None: panel.add_argument("--output", type=Path, default=None, help="Écrire le rapport markdown ici au lieu de stdout") panel.add_argument("--hide-g1", action="store_true", help="Masquer les écarts mineurs (G1) dans le détail du consensus") + results = subparsers.add_parser("results", help="Gérer le dataset de résultats (transcripts locaux -> Hugging Face)") + results_subparsers = results.add_subparsers(dest="results_command") + results_push = results_subparsers.add_parser( + "push", + help="Pousser les transcripts locaux vers le dataset de résultats (officiels uniquement)", + ) + results_push.add_argument("--transcriptions-dir", type=Path, default=RESULTS_DEFAULT_TRANSCRIPTIONS_DIR, help="Dossier des transcripts locaux") + results_push.add_argument("--include", default=None, help="Ne garder que les fichiers dont le nom contient cette chaîne (ex: whisperx, kyutai, cohere)") + results_push.add_argument("--corpus", default=None, help=f"Dataset corpus (allowlist des officiels); défaut $EVAL_CORPUS_REPO ou {DEFAULT_CORPUS_REPO}") + results_push.add_argument("--results", dest="results_repo", default=None, help=f"Dataset résultats; défaut $EVAL_RESULTS_REPO ou {DEFAULT_RESULTS_REPO}") + results_push.add_argument("--token", default=None, help="Token Hugging Face write; défaut $HF_TOKEN") + results_push.add_argument("--message", default="Push local transcripts (officiels only)", help="Message de commit") + results_push.add_argument("--dry-run", action="store_true", help="Afficher le plan sans rien pousser") + args = parser.parse_args() try: @@ -422,7 +443,28 @@ def main() -> None: ) ) return - except (FileNotFoundError, DataMigrationError, ScoringError, JudgeCliError, JudgeError, AlbertError, ScalewayError, ElevenLabsError, OmlxError, httpx.HTTPError) as exc: + + if args.command == "results" and args.results_command == "push": + client = ResultsClient( + corpus_repo=args.corpus, + results_repo=args.results_repo, + token=args.token, + ) + plan = client.plan(transcriptions_dir=args.transcriptions_dir, include=args.include) + print(f"Corpus public ({len(plan.officiels)} samples): {', '.join(plan.officiels)}") + print(f"À pousser : {len(plan.uploads)} fichier(s) vers {plan.results_repo}") + for u in plan.uploads: + print(f" + {u.path_in_repo}") + if plan.skipped_samples: + print(f"Ignorés (hors corpus public): {', '.join(plan.skipped_samples)}") + if args.dry_run: + print("[dry-run] relancer sans --dry-run pour pousser.") + return + client.push(plan, message=args.message) + if plan.uploads: + print(f"Poussé {len(plan.uploads)} fichier(s) vers {plan.results_repo}") + return + except (FileNotFoundError, DataMigrationError, ScoringError, JudgeCliError, JudgeError, AlbertError, ScalewayError, ElevenLabsError, OmlxError, ResultsError, httpx.HTTPError) as exc: print(f"Error: {exc}", file=sys.stderr) sys.exit(1) @@ -440,5 +482,7 @@ def main() -> None: elevenlabs.print_help() elif args.command == "omlx": omlx.print_help() + elif args.command == "results": + results.print_help() else: parser.print_help() diff --git a/src/eval_transcript/results.py b/src/eval_transcript/results.py new file mode 100644 index 0000000..cf2dd6e --- /dev/null +++ b/src/eval_transcript/results.py @@ -0,0 +1,200 @@ +"""Push des transcripts locaux vers le dataset de resultats Hugging Face. + +Sert a verser sur le dataset de resultats les sorties des modeles tournant en +local (WhisperX, Kyutai, Cohere via MLX), qui ne passent pas par la CI remote. + +Garde-fou de souverainete / RGPD : seuls les transcripts des samples presents +dans le corpus PUBLIC (cote ``ground_truth/``) sont pousses. Un sample absent du +corpus public (reunion interne, sample retire) est ignore -> aucune donnee privee +ne part sur le Hub. L'allowlist est derivee du corpus live, pas codee en dur. +""" + +from __future__ import annotations + +import os +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + + +DEFAULT_CORPUS_REPO = "AgentPublic/eval-stt-officiels" +DEFAULT_RESULTS_REPO = "AgentPublic/eval-stt-results" +DEFAULT_TRANSCRIPTIONS_DIR = Path("data/transcriptions") + +CORPUS_REPO_ENV = "EVAL_CORPUS_REPO" +RESULTS_REPO_ENV = "EVAL_RESULTS_REPO" +TOKEN_ENV = "HF_TOKEN" + +DATASET_REPO_TYPE = "dataset" +GROUND_TRUTH_REPO_SUBDIR = "ground_truth" +TRANSCRIPTIONS_REPO_SUBDIR = "transcriptions" +TRANSCRIPT_SUFFIX = ".txt" + + +class ResultsError(RuntimeError): + """Raised when local results cannot be pushed safely.""" + + +def _hf_http_error_types() -> tuple[type[BaseException], ...]: + """huggingface_hub HTTP error class, or an empty tuple (catches nothing) if absent. + + Lets the live calls surface a clean ``ResultsError`` (e.g. invalid token, repo + not found) instead of a raw traceback, while keeping the unit tests free of a + hard huggingface_hub dependency. + """ + try: + from huggingface_hub.errors import HfHubHTTPError + except ImportError: # pragma: no cover - exercised only in production + return () + return (HfHubHTTPError,) + + +class HfApiLike(Protocol): + """Subset of huggingface_hub.HfApi used here (keeps unit tests dependency-free).""" + + def list_repo_files( + self, repo_id: str, *, repo_type: str = ..., revision: str | None = None + ) -> list[str]: ... + + def create_commit( + self, + *, + repo_id: str, + operations: Sequence[Any], + commit_message: str, + repo_type: str = ..., + token: str | None = None, + ) -> Any: ... + + +@dataclass(frozen=True) +class PlannedUpload: + sample_id: str + path_in_repo: str + local_path: Path + + +@dataclass(frozen=True) +class PushPlan: + results_repo: str + officiels: list[str] + uploads: list[PlannedUpload] + skipped_samples: list[str] + + +def public_sample_ids(api: HfApiLike, corpus_repo: str, *, revision: str | None = None) -> set[str]: + """Sample IDs declared public by the corpus dataset (one per ``ground_truth/`` file).""" + try: + files = api.list_repo_files(corpus_repo, repo_type=DATASET_REPO_TYPE, revision=revision) + except _hf_http_error_types() as exc: + raise ResultsError(f"Failed to read corpus dataset {corpus_repo}: {exc}") from exc + prefix = f"{GROUND_TRUTH_REPO_SUBDIR}/" + return { + Path(f[len(prefix):]).stem + for f in files + if f.startswith(prefix) and f.endswith(TRANSCRIPT_SUFFIX) + } + + +def build_push_plan( + *, + api: HfApiLike, + corpus_repo: str, + results_repo: str, + transcriptions_dir: Path, + include: str | None = None, +) -> PushPlan: + """Plan the upload, keeping only transcripts of samples in the public corpus.""" + officiels = public_sample_ids(api, corpus_repo) + if not officiels: + raise ResultsError(f"No public samples found in corpus repo {corpus_repo}") + if not transcriptions_dir.exists(): + raise ResultsError(f"Transcriptions directory not found: {transcriptions_dir}") + + uploads: list[PlannedUpload] = [] + skipped: set[str] = set() + for sample_dir in sorted(p for p in transcriptions_dir.iterdir() if p.is_dir()): + sid = sample_dir.name + if sid not in officiels: + skipped.add(sid) + continue + for txt in sorted(sample_dir.glob(f"*{TRANSCRIPT_SUFFIX}")): + if include and include not in txt.name: + continue + uploads.append( + PlannedUpload( + sample_id=sid, + path_in_repo=f"{TRANSCRIPTIONS_REPO_SUBDIR}/{sid}/{txt.name}", + local_path=txt, + ) + ) + return PushPlan( + results_repo=results_repo, + officiels=sorted(officiels), + uploads=uploads, + skipped_samples=sorted(skipped), + ) + + +class ResultsClient: + """Reads the public corpus and pushes local transcripts to the results dataset.""" + + def __init__( + self, + *, + corpus_repo: str | None = None, + results_repo: str | None = None, + token: str | None = None, + api: HfApiLike | None = None, + ) -> None: + self.corpus_repo = corpus_repo or os.getenv(CORPUS_REPO_ENV, DEFAULT_CORPUS_REPO) + self.results_repo = results_repo or os.getenv(RESULTS_REPO_ENV, DEFAULT_RESULTS_REPO) + self.token = token if token is not None else os.getenv(TOKEN_ENV) + self._api = api + + def _get_api(self) -> HfApiLike: + if self._api is not None: + return self._api + try: + from huggingface_hub import HfApi # type: ignore[import-not-found] + except ImportError as exc: # pragma: no cover - exercised only in production + raise ResultsError("huggingface_hub is required for live HF Hub operations") from exc + return HfApi(token=self.token) + + def plan( + self, *, transcriptions_dir: Path = DEFAULT_TRANSCRIPTIONS_DIR, include: str | None = None + ) -> PushPlan: + return build_push_plan( + api=self._get_api(), + corpus_repo=self.corpus_repo, + results_repo=self.results_repo, + transcriptions_dir=transcriptions_dir, + include=include, + ) + + def push(self, plan: PushPlan, *, message: str = "Push local transcripts (officiels only)") -> None: + if not plan.uploads: + return + if not self.token: + raise ResultsError( + "HF_TOKEN is required to push results; set the HF_TOKEN environment variable or pass token=..." + ) + try: + from huggingface_hub import CommitOperationAdd # type: ignore[import-not-found] + except ImportError as exc: # pragma: no cover - exercised only in production + raise ResultsError("huggingface_hub is required for live HF Hub operations") from exc + operations = [ + CommitOperationAdd(path_in_repo=u.path_in_repo, path_or_fileobj=str(u.local_path)) + for u in plan.uploads + ] + try: + self._get_api().create_commit( + repo_id=plan.results_repo, + repo_type=DATASET_REPO_TYPE, + operations=operations, + commit_message=message, + token=self.token, + ) + except _hf_http_error_types() as exc: + raise ResultsError(f"Failed to push to results dataset {plan.results_repo}: {exc}") from exc diff --git a/tests/test_results.py b/tests/test_results.py new file mode 100644 index 0000000..5b24fe6 --- /dev/null +++ b/tests/test_results.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import contextlib +import io +import os +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from huggingface_hub.errors import HfHubHTTPError + +import eval_transcript +from eval_transcript.results import ( + PlannedUpload, + PushPlan, + ResultsClient, + ResultsError, + build_push_plan, + public_sample_ids, +) + + +class FakeHfApi: + def __init__( + self, + *, + ground_truth_ids: list[str], + list_error: BaseException | None = None, + commit_error: BaseException | None = None, + ) -> None: + self.files = [f"ground_truth/{sid}.txt" for sid in ground_truth_ids] + [ + f"audio/{sid}.mp3" for sid in ground_truth_ids + ] + self.list_error = list_error + self.commit_error = commit_error + self.commit_calls: list[dict[str, object]] = [] + + def list_repo_files(self, repo_id, *, repo_type, revision=None): + if self.list_error is not None: + raise self.list_error + return list(self.files) + + def create_commit(self, *, repo_id, operations, commit_message, repo_type, token=None): + if self.commit_error is not None: + raise self.commit_error + self.commit_calls.append( + { + "repo_id": repo_id, + "ops": [(op.path_in_repo, str(op.path_or_fileobj)) for op in operations], + "message": commit_message, + "repo_type": repo_type, + "token": token, + } + ) + + +def _make_transcriptions(root: Path, layout: dict[str, list[str]]) -> Path: + tx = root / "transcriptions" + for sid, files in layout.items(): + d = tx / sid + d.mkdir(parents=True) + for name in files: + (d / name).write_text("bonjour", encoding="utf-8") + return tx + + +class PublicSampleIdsTests(unittest.TestCase): + def test_extracts_ids_from_ground_truth(self) -> None: + api = FakeHfApi(ground_truth_ids=["officiel-a", "officiel-b"]) + self.assertEqual(public_sample_ids(api, "corpus"), {"officiel-a", "officiel-b"}) + + +class BuildPushPlanTests(unittest.TestCase): + def test_keeps_officiels_and_skips_private(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + tx = _make_transcriptions( + root, + { + "officiel-a": ["whisperx__large-v2.txt", "omlx__cohere.txt"], + "reunion-interne": ["whisperx__large-v2.txt"], # privé -> ignoré + }, + ) + api = FakeHfApi(ground_truth_ids=["officiel-a"]) + plan = build_push_plan( + api=api, corpus_repo="c", results_repo="r", transcriptions_dir=tx + ) + paths = sorted(u.path_in_repo for u in plan.uploads) + self.assertEqual( + paths, + [ + "transcriptions/officiel-a/omlx__cohere.txt", + "transcriptions/officiel-a/whisperx__large-v2.txt", + ], + ) + self.assertEqual(plan.skipped_samples, ["reunion-interne"]) + + def test_include_filter(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + tx = _make_transcriptions( + root, {"officiel-a": ["whisperx__large-v2.txt", "omlx__cohere.txt"]} + ) + api = FakeHfApi(ground_truth_ids=["officiel-a"]) + plan = build_push_plan( + api=api, corpus_repo="c", results_repo="r", transcriptions_dir=tx, include="whisperx" + ) + self.assertEqual( + [u.path_in_repo for u in plan.uploads], + ["transcriptions/officiel-a/whisperx__large-v2.txt"], + ) + + def test_empty_corpus_raises(self) -> None: + with TemporaryDirectory() as tmp: + tx = _make_transcriptions(Path(tmp), {"officiel-a": ["whisperx__large-v2.txt"]}) + api = FakeHfApi(ground_truth_ids=[]) + with self.assertRaisesRegex(ResultsError, "No public samples"): + build_push_plan(api=api, corpus_repo="c", results_repo="r", transcriptions_dir=tx) + + def test_missing_dir_raises(self) -> None: + api = FakeHfApi(ground_truth_ids=["officiel-a"]) + with self.assertRaisesRegex(ResultsError, "not found"): + build_push_plan( + api=api, corpus_repo="c", results_repo="r", transcriptions_dir=Path("/no/such/dir") + ) + + +class PushTests(unittest.TestCase): + def test_push_requires_token(self) -> None: + plan = PushPlan( + results_repo="r", + officiels=["officiel-a"], + uploads=[ + PlannedUpload( + "officiel-a", "transcriptions/officiel-a/whisperx__large-v2.txt", Path("x.txt") + ) + ], + skipped_samples=[], + ) + with patch.dict("os.environ", {}, clear=True): + client = ResultsClient(api=FakeHfApi(ground_truth_ids=["officiel-a"])) + with self.assertRaisesRegex(ResultsError, "HF_TOKEN"): + client.push(plan) + + def test_push_builds_single_commit(self) -> None: + # huggingface_hub.CommitOperationAdd réel expose .path_in_repo / .path_or_fileobj, + # compatibles avec FakeHfApi -> pas besoin de le patcher. + with TemporaryDirectory() as tmp: + root = Path(tmp) + tx = _make_transcriptions(root, {"officiel-a": ["whisperx__large-v2.txt"]}) + api = FakeHfApi(ground_truth_ids=["officiel-a"]) + client = ResultsClient(results_repo="team/results", token="tok", api=api) + plan = client.plan(transcriptions_dir=tx) + client.push(plan, message="msg") + self.assertEqual(len(api.commit_calls), 1) + call = api.commit_calls[0] + self.assertEqual(call["repo_id"], "team/results") + self.assertEqual(call["token"], "tok") + self.assertEqual( + call["ops"], + [ + ( + "transcriptions/officiel-a/whisperx__large-v2.txt", + str(tx / "officiel-a" / "whisperx__large-v2.txt"), + ) + ], + ) + + +class ResultsCliTests(unittest.TestCase): + def test_dry_run_prints_plan_without_pushing(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + tx = _make_transcriptions( + root, {"officiel-a": ["whisperx__large-v2.txt"], "reunion-x": ["whisperx__large-v2.txt"]} + ) + api = FakeHfApi(ground_truth_ids=["officiel-a"]) + argv = [ + "eval-transcript", "results", "push", + "--transcriptions-dir", str(tx), + "--dry-run", + ] + stdout, stderr = io.StringIO(), io.StringIO() + with patch("eval_transcript.ResultsClient") as client_cls: + client = client_cls.return_value + client.plan.return_value = build_push_plan( + api=api, corpus_repo="c", results_repo="team/results", transcriptions_dir=tx + ) + with patch.dict(os.environ, {}, clear=True), patch.object(sys, "argv", argv), \ + contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + eval_transcript.main() + out = stdout.getvalue() + self.assertIn("transcriptions/officiel-a/whisperx__large-v2.txt", out) + self.assertIn("Ignorés (hors corpus public): reunion-x", out) + self.assertIn("[dry-run]", out) + client.push.assert_not_called() + + +class HfErrorTests(unittest.TestCase): + def test_corpus_read_error_becomes_results_error(self) -> None: + with TemporaryDirectory() as tmp: + tx = _make_transcriptions(Path(tmp), {"officiel-a": ["whisperx__large-v2.txt"]}) + api = FakeHfApi(ground_truth_ids=["officiel-a"], list_error=HfHubHTTPError("403 Forbidden")) + with self.assertRaisesRegex(ResultsError, "Failed to read corpus"): + build_push_plan(api=api, corpus_repo="c", results_repo="r", transcriptions_dir=tx) + + def test_push_commit_error_becomes_results_error(self) -> None: + with TemporaryDirectory() as tmp: + tx = _make_transcriptions(Path(tmp), {"officiel-a": ["whisperx__large-v2.txt"]}) + api = FakeHfApi(ground_truth_ids=["officiel-a"], commit_error=HfHubHTTPError("401 Unauthorized")) + client = ResultsClient(results_repo="r", token="tok", api=api) + plan = client.plan(transcriptions_dir=tx) + with self.assertRaisesRegex(ResultsError, "Failed to push"): + client.push(plan) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 21d2039..d581378 100644 --- a/uv.lock +++ b/uv.lock @@ -501,6 +501,7 @@ source = { editable = "." } dependencies = [ { name = "elevenlabs" }, { name = "httpx" }, + { name = "huggingface-hub" }, { name = "jiwer" }, { name = "python-dotenv" }, { name = "text2num" }, @@ -518,6 +519,7 @@ whisperx = [ requires-dist = [ { name = "elevenlabs", specifier = ">=2.50.0" }, { name = "httpx", specifier = ">=0.28.1" }, + { name = "huggingface-hub", specifier = ">=0.24.0" }, { name = "jiwer", specifier = ">=4.0.0" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "text2num", specifier = ">=2.4.0" },