From 42951cdb8bc416ef282d75a4966ffc2841d82b21 Mon Sep 17 00:00:00 2001 From: benoitvx Date: Thu, 18 Jun 2026 17:23:07 +0200 Subject: [PATCH] =?UTF-8?q?ci:=20bench=20remote=20(officiels)=20=E2=80=94?= =?UTF-8?q?=20transcription=20API=20+=20WER?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workflow workflow_dispatch : pull corpus public HF, transcription des modeles remote (Albert, Voxtral/Scaleway, ElevenLabs en option), scoring WER, push des resultats vers AgentPublic/eval-stt-results. Modeles locaux exclus (phase 2, runner self-hosted). Skips Voxtral temporaires (> 28 min + silence connu). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/bench-remote.yml | 131 +++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 .github/workflows/bench-remote.yml diff --git a/.github/workflows/bench-remote.yml b/.github/workflows/bench-remote.yml new file mode 100644 index 0000000..25b43b2 --- /dev/null +++ b/.github/workflows/bench-remote.yml @@ -0,0 +1,131 @@ +name: Bench remote (officiels) + +# Transcription des discours officiels (corpus public HF) par les modeles accessibles +# via API, puis calcul du WER. Declenchement manuel uniquement (chaque run = appels API +# facturables). Les modeles locaux (WhisperX, Kyutai, Cohere/MLX) ne sont PAS ici : ils +# exigent un GPU ou MLX et passeront par un runner self-hosted (phase 2). + +on: + workflow_dispatch: + inputs: + providers: + description: "Providers remote (csv) : albert, scaleway, elevenlabs" + default: "albert,scaleway" + push_results: + description: "Pousser les resultats vers AgentPublic/eval-stt-results (HF)" + type: boolean + default: true + +permissions: + contents: read + +jobs: + bench: + runs-on: ubuntu-latest + env: + ALBERT_BASE_URL: https://albert.api.etalab.gouv.fr/v1 + ALBERT_API_KEY: ${{ secrets.ALBERT_API_KEY }} + SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }} + SCW_DEFAULT_PROJECT_ID: ${{ secrets.SCW_DEFAULT_PROJECT_ID }} + ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + CORPUS: AgentPublic/eval-stt-officiels + RESULTS: AgentPublic/eval-stt-results + PROVIDERS_INPUT: ${{ inputs.providers }} + # Voxtral (Scaleway) : limite 30 min/fichier -> on saute au-dela de ~28 min (auto, + # via ffprobe). Quand un long sample repasse sous la limite, il est couvert sans + # toucher la CI. crise_energetique : collapse sur un silence long de ~35 s (cas + # connu) -> skip explicite. Exclusions TEMPORAIRES : le chunk VAD est a porter dans + # le provider scaleway (cf. reprise PR #36). + VOXTRAL_MAX_SECONDS: "1700" + VOXTRAL_SILENCE_SKIP: "crise_energetique_aides_mai" + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Dependances systeme (ffprobe) + run: sudo apt-get update && sudo apt-get install -y ffmpeg + + - name: Install projet + run: uv sync + + - name: Pull corpus public depuis HF (lecture anonyme) + run: | + uvx --from "huggingface_hub[cli]" hf download "$CORPUS" \ + --repo-type dataset --local-dir hf-corpus + mkdir -p data/audio data/ground_truth + cp hf-corpus/audio/* data/audio/ + cp hf-corpus/ground_truth/* data/ground_truth/ + echo "Corpus: $(ls data/audio/*.mp3 | wc -l) audios, $(ls data/ground_truth/*.txt | wc -l) references" + + - name: Transcription (modeles remote) + run: | + set -uo pipefail + IFS=',' read -ra PROVIDERS <<< "$PROVIDERS_INPUT" + for audio in data/audio/*.mp3; do + id="$(basename "$audio" .mp3)" + dur="$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$audio" | cut -d. -f1)" + for p in "${PROVIDERS[@]}"; do + case "$p" in + albert) + uv run eval-transcript albert transcribe "$audio" --language fr --timeout 900 --save \ + || echo "::warning::albert a echoue sur $id" ;; + scaleway) + if [ "${dur:-0}" -gt "$VOXTRAL_MAX_SECONDS" ]; then + echo "skip voxtral (> 28 min): $id"; continue + fi + if echo "$VOXTRAL_SILENCE_SKIP" | grep -qw "$id"; then + echo "skip voxtral (silence long connu): $id"; continue + fi + uv run eval-transcript scaleway transcribe "$audio" --language fr --timeout 900 --save \ + || echo "::warning::voxtral a echoue sur $id" ;; + elevenlabs) + uv run eval-transcript elevenlabs transcribe "$audio" --language fr --timeout 900 --save \ + || echo "::warning::elevenlabs a echoue sur $id" ;; + *) + echo "::warning::provider inconnu: $p" ;; + esac + done + done + + - name: Scoring WER + run: | + mkdir -p results/reports + REV="$(uvx --from huggingface_hub python -c "from huggingface_hub import HfApi; print(HfApi().dataset_info('$CORPUS').sha)")" + STAMP="$(date -u +%Y-%m-%dT%H-%M-%SZ)" + REPORT="results/reports/${STAMP}__wer.md" + { + echo "# WER - run CI $STAMP" + echo + echo "- corpus: $CORPUS @ ${REV}" + echo "- providers: $PROVIDERS_INPUT" + echo + } > "$REPORT" + uv run eval-transcript score all --normalization standard_numbers >> "$REPORT" + cat "$REPORT" + + - name: Resultats vers HF (transcripts officiels + rapport) + if: ${{ inputs.push_results }} + run: | + if [ -z "${HF_TOKEN:-}" ]; then + echo "HF_TOKEN absent -> push HF saute, artefacts uniquement"; exit 0 + fi + # En CI le corpus tire = officiels uniquement, donc data/transcriptions ne + # contient que du public. Garde-fou implicite : aucune reunion interne ici. + mkdir -p results/transcriptions + cp -r data/transcriptions/* results/transcriptions/ 2>/dev/null || true + uvx --from "huggingface_hub[cli]" hf upload "$RESULTS" results . \ + --repo-type dataset --commit-message "CI run $(date -u +%FT%TZ)" \ + || echo "::warning::push HF des resultats a echoue" + + - name: Artefacts (toujours) + if: always() + uses: actions/upload-artifact@v4 + with: + name: bench-remote + path: | + data/transcriptions/** + results/**