feat(results): sous-commande results push (transcripts locaux → HF)

README.md

@@ -399,3 +399,18 @@ Set these as repository secrets (Settings > Secrets and variables > Actions):
 ### Running
 
 Trigger from the Actions tab (Run workflow, branch `main`). The bench accepts a `providers` input (default `albert,scaleway`); the judge accepts `mode` (`single`/`panel`) and `passes`. Each run consumes billable API calls, so the workflows are manual by design. Outputs are also uploaded as workflow artifacts.
+
+### Pushing local transcripts to the results dataset
+
+The CI only covers the API models. Local models (WhisperX, Kyutai, Cohere via MLX) run on your machine; push their outputs to the results dataset so the WER and judge cover every model:
+
+```bash
+# Dry-run: show what would be pushed (and what is skipped)
+uv run eval-transcript results push --dry-run
+
+# Push for real (single commit); optionally filter by file name
+uv run eval-transcript results push
+uv run eval-transcript results push --include whisperx
+```
+
+Safety: the command derives the public allowlist from the **corpus** dataset (`ground_truth/<id>`) and uploads only transcripts whose sample is in it. Any local transcript for a sample absent from the public corpus (internal meeting, removed sample) is skipped, so private data never reaches the Hub. Override the repos with `--corpus` / `--results` (or `EVAL_CORPUS_REPO` / `EVAL_RESULTS_REPO`); pushing needs `HF_TOKEN` with write access.

pyproject.toml

@@ -10,6 +10,7 @@ requires-python = ">=3.12"
 dependencies = [
     "elevenlabs>=2.50.0",
     "httpx>=0.28.1",
+    "huggingface-hub>=0.24.0",
     "jiwer>=4.0.0",
     "python-dotenv>=1.2.2",
     "text2num>=2.4.0",

src/eval_transcript/__init__.py

@@ -59,6 +59,13 @@
     write_or_print_score_output,
 )
 from eval_transcript.transcriptions import TranscriptionOutput, print_transcription_output, transcription_text
+from eval_transcript.results import (
+    DEFAULT_CORPUS_REPO,
+    DEFAULT_RESULTS_REPO,
+    DEFAULT_TRANSCRIPTIONS_DIR as RESULTS_DEFAULT_TRANSCRIPTIONS_DIR,
+    ResultsClient,
+    ResultsError,
+)
 
 
 DEPRECATED_SOURCE_TRUTH_FLAG_MESSAGE = (
@@ -210,6 +217,20 @@ def main() -> None:
     panel.add_argument("--output", type=Path, default=None, help="Écrire le rapport markdown ici au lieu de stdout")
     panel.add_argument("--hide-g1", action="store_true", help="Masquer les écarts mineurs (G1) dans le détail du consensus")
 
+    results = subparsers.add_parser("results", help="Gérer le dataset de résultats (transcripts locaux -> Hugging Face)")
+    results_subparsers = results.add_subparsers(dest="results_command")
+    results_push = results_subparsers.add_parser(
+        "push",
+        help="Pousser les transcripts locaux vers le dataset de résultats (officiels uniquement)",
+    )
+    results_push.add_argument("--transcriptions-dir", type=Path, default=RESULTS_DEFAULT_TRANSCRIPTIONS_DIR, help="Dossier des transcripts locaux")
+    results_push.add_argument("--include", default=None, help="Ne garder que les fichiers dont le nom contient cette chaîne (ex: whisperx, kyutai, cohere)")
+    results_push.add_argument("--corpus", default=None, help=f"Dataset corpus (allowlist des officiels); défaut $EVAL_CORPUS_REPO ou {DEFAULT_CORPUS_REPO}")
+    results_push.add_argument("--results", dest="results_repo", default=None, help=f"Dataset résultats; défaut $EVAL_RESULTS_REPO ou {DEFAULT_RESULTS_REPO}")
+    results_push.add_argument("--token", default=None, help="Token Hugging Face write; défaut $HF_TOKEN")
+    results_push.add_argument("--message", default="Push local transcripts (officiels only)", help="Message de commit")
+    results_push.add_argument("--dry-run", action="store_true", help="Afficher le plan sans rien pousser")
+
     args = parser.parse_args()
 
     try:
@@ -422,7 +443,28 @@ def main() -> None:
                 )
             )
             return
-    except (FileNotFoundError, DataMigrationError, ScoringError, JudgeCliError, JudgeError, AlbertError, ScalewayError, ElevenLabsError, OmlxError, httpx.HTTPError) as exc:
+
+        if args.command == "results" and args.results_command == "push":
+            client = ResultsClient(
+                corpus_repo=args.corpus,
+                results_repo=args.results_repo,
+                token=args.token,
+            )
+            plan = client.plan(transcriptions_dir=args.transcriptions_dir, include=args.include)
+            print(f"Corpus public ({len(plan.officiels)} samples): {', '.join(plan.officiels)}")
+            print(f"À pousser : {len(plan.uploads)} fichier(s) vers {plan.results_repo}")
+            for u in plan.uploads:
+                print(f"  + {u.path_in_repo}")
+            if plan.skipped_samples:
+                print(f"Ignorés (hors corpus public): {', '.join(plan.skipped_samples)}")
+            if args.dry_run:
+                print("[dry-run] relancer sans --dry-run pour pousser.")
+                return
+            client.push(plan, message=args.message)
+            if plan.uploads:
+                print(f"Poussé {len(plan.uploads)} fichier(s) vers {plan.results_repo}")
+            return
+    except (FileNotFoundError, DataMigrationError, ScoringError, JudgeCliError, JudgeError, AlbertError, ScalewayError, ElevenLabsError, OmlxError, ResultsError, httpx.HTTPError) as exc:
         print(f"Error: {exc}", file=sys.stderr)
         sys.exit(1)
 
@@ -440,5 +482,7 @@ def main() -> None:
         elevenlabs.print_help()
     elif args.command == "omlx":
         omlx.print_help()
+    elif args.command == "results":
+        results.print_help()
     else:
         parser.print_help()

src/eval_transcript/results.py

@@ -0,0 +1,200 @@
+"""Push des transcripts locaux vers le dataset de resultats Hugging Face.
+
+Sert a verser sur le dataset de resultats les sorties des modeles tournant en
+local (WhisperX, Kyutai, Cohere via MLX), qui ne passent pas par la CI remote.
+
+Garde-fou de souverainete / RGPD : seuls les transcripts des samples presents
+dans le corpus PUBLIC (cote ``ground_truth/``) sont pousses. Un sample absent du
+corpus public (reunion interne, sample retire) est ignore -> aucune donnee privee
+ne part sur le Hub. L'allowlist est derivee du corpus live, pas codee en dur.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Protocol
+
+
+DEFAULT_CORPUS_REPO = "AgentPublic/eval-stt-officiels"
+DEFAULT_RESULTS_REPO = "AgentPublic/eval-stt-results"
+DEFAULT_TRANSCRIPTIONS_DIR = Path("data/transcriptions")
+
+CORPUS_REPO_ENV = "EVAL_CORPUS_REPO"
+RESULTS_REPO_ENV = "EVAL_RESULTS_REPO"
+TOKEN_ENV = "HF_TOKEN"
+
+DATASET_REPO_TYPE = "dataset"
+GROUND_TRUTH_REPO_SUBDIR = "ground_truth"
+TRANSCRIPTIONS_REPO_SUBDIR = "transcriptions"
+TRANSCRIPT_SUFFIX = ".txt"
+
+
+class ResultsError(RuntimeError):
+    """Raised when local results cannot be pushed safely."""
+
+
+def _hf_http_error_types() -> tuple[type[BaseException], ...]:
+    """huggingface_hub HTTP error class, or an empty tuple (catches nothing) if absent.
+
+    Lets the live calls surface a clean ``ResultsError`` (e.g. invalid token, repo
+    not found) instead of a raw traceback, while keeping the unit tests free of a
+    hard huggingface_hub dependency.
+    """
+    try:
+        from huggingface_hub.errors import HfHubHTTPError
+    except ImportError:  # pragma: no cover - exercised only in production
+        return ()
+    return (HfHubHTTPError,)
+
+
+class HfApiLike(Protocol):
+    """Subset of huggingface_hub.HfApi used here (keeps unit tests dependency-free)."""
+
+    def list_repo_files(
+        self, repo_id: str, *, repo_type: str = ..., revision: str | None = None
+    ) -> list[str]: ...
+
+    def create_commit(
+        self,
+        *,
+        repo_id: str,
+        operations: Sequence[Any],
+        commit_message: str,
+        repo_type: str = ...,
+        token: str | None = None,
+    ) -> Any: ...
+
+
+@dataclass(frozen=True)
+class PlannedUpload:
+    sample_id: str
+    path_in_repo: str
+    local_path: Path
+
+
+@dataclass(frozen=True)
+class PushPlan:
+    results_repo: str
+    officiels: list[str]
+    uploads: list[PlannedUpload]
+    skipped_samples: list[str]
+
+
+def public_sample_ids(api: HfApiLike, corpus_repo: str, *, revision: str | None = None) -> set[str]:
+    """Sample IDs declared public by the corpus dataset (one per ``ground_truth/<id>`` file)."""
+    try:
+        files = api.list_repo_files(corpus_repo, repo_type=DATASET_REPO_TYPE, revision=revision)
+    except _hf_http_error_types() as exc:
+        raise ResultsError(f"Failed to read corpus dataset {corpus_repo}: {exc}") from exc
+    prefix = f"{GROUND_TRUTH_REPO_SUBDIR}/"
+    return {
+        Path(f[len(prefix):]).stem
+        for f in files
+        if f.startswith(prefix) and f.endswith(TRANSCRIPT_SUFFIX)
+    }
+
+
+def build_push_plan(
+    *,
+    api: HfApiLike,
+    corpus_repo: str,
+    results_repo: str,
+    transcriptions_dir: Path,
+    include: str | None = None,
+) -> PushPlan:
+    """Plan the upload, keeping only transcripts of samples in the public corpus."""
+    officiels = public_sample_ids(api, corpus_repo)
+    if not officiels:
+        raise ResultsError(f"No public samples found in corpus repo {corpus_repo}")
+    if not transcriptions_dir.exists():
+        raise ResultsError(f"Transcriptions directory not found: {transcriptions_dir}")
+
+    uploads: list[PlannedUpload] = []
+    skipped: set[str] = set()
+    for sample_dir in sorted(p for p in transcriptions_dir.iterdir() if p.is_dir()):
+        sid = sample_dir.name
+        if sid not in officiels:
+            skipped.add(sid)
+            continue
+        for txt in sorted(sample_dir.glob(f"*{TRANSCRIPT_SUFFIX}")):
+            if include and include not in txt.name:
+                continue
+            uploads.append(
+                PlannedUpload(
+                    sample_id=sid,
+                    path_in_repo=f"{TRANSCRIPTIONS_REPO_SUBDIR}/{sid}/{txt.name}",
+                    local_path=txt,
+                )
+            )
+    return PushPlan(
+        results_repo=results_repo,
+        officiels=sorted(officiels),
+        uploads=uploads,
+        skipped_samples=sorted(skipped),
+    )
+
+
+class ResultsClient:
+    """Reads the public corpus and pushes local transcripts to the results dataset."""
+
+    def __init__(
+        self,
+        *,
+        corpus_repo: str | None = None,
+        results_repo: str | None = None,
+        token: str | None = None,
+        api: HfApiLike | None = None,
+    ) -> None:
+        self.corpus_repo = corpus_repo or os.getenv(CORPUS_REPO_ENV, DEFAULT_CORPUS_REPO)
+        self.results_repo = results_repo or os.getenv(RESULTS_REPO_ENV, DEFAULT_RESULTS_REPO)
+        self.token = token if token is not None else os.getenv(TOKEN_ENV)
+        self._api = api
+
+    def _get_api(self) -> HfApiLike:
+        if self._api is not None:
+            return self._api
+        try:
+            from huggingface_hub import HfApi  # type: ignore[import-not-found]
+        except ImportError as exc:  # pragma: no cover - exercised only in production
+            raise ResultsError("huggingface_hub is required for live HF Hub operations") from exc
+        return HfApi(token=self.token)
+
+    def plan(
+        self, *, transcriptions_dir: Path = DEFAULT_TRANSCRIPTIONS_DIR, include: str | None = None
+    ) -> PushPlan:
+        return build_push_plan(
+            api=self._get_api(),
+            corpus_repo=self.corpus_repo,
+            results_repo=self.results_repo,
+            transcriptions_dir=transcriptions_dir,
+            include=include,
+        )
+
+    def push(self, plan: PushPlan, *, message: str = "Push local transcripts (officiels only)") -> None:
+        if not plan.uploads:
+            return
+        if not self.token:
+            raise ResultsError(
+                "HF_TOKEN is required to push results; set the HF_TOKEN environment variable or pass token=..."
+            )
+        try:
+            from huggingface_hub import CommitOperationAdd  # type: ignore[import-not-found]
+        except ImportError as exc:  # pragma: no cover - exercised only in production
+            raise ResultsError("huggingface_hub is required for live HF Hub operations") from exc
+        operations = [
+            CommitOperationAdd(path_in_repo=u.path_in_repo, path_or_fileobj=str(u.local_path))
+            for u in plan.uploads
+        ]
+        try:
+            self._get_api().create_commit(
+                repo_id=plan.results_repo,
+                repo_type=DATASET_REPO_TYPE,
+                operations=operations,
+                commit_message=message,
+                token=self.token,
+            )
+        except _hf_http_error_types() as exc:
+            raise ResultsError(f"Failed to push to results dataset {plan.results_repo}: {exc}") from exc