From 4b1668079c1394864802fae6a66a7e9c2797a531 Mon Sep 17 00:00:00 2001 From: benoitvx Date: Tue, 23 Jun 2026 17:45:17 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(ci):=20judge.yml=20=E2=80=94=20inputs?= =?UTF-8?q?=20judge=5Fprovider=20+=20judge=5Fmodel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Permet de lancer le LLM-as-a-judge en CI avec un modèle OpenRouter au choix (ex. anthropic/claude-opus-4.8) au lieu du seul défaut. single respecte --judge-provider/--judge-model ; panel accepte un modèle openrouter custom. Le nom du rapport inclut un slug du modèle → pas d'écrasement du run Mistral (gravite-single.md vs gravite-single-anthropic_claude-opus-4.8.md). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/judge.yml | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/judge.yml b/.github/workflows/judge.yml index 98855dd..c75a2f6 100644 --- a/.github/workflows/judge.yml +++ b/.github/workflows/judge.yml @@ -12,10 +12,18 @@ on: workflow_dispatch: inputs: mode: - description: "single (un juge, Albert/Mistral) ou panel (multi-juges + consensus)" + description: "single (un juge) ou panel (multi-juges + consensus)" type: choice options: [single, panel] default: single + judge_provider: + description: "single uniquement : fournisseur du juge" + type: choice + options: [albert, openrouter] + default: albert + judge_model: + description: "Modele juge (override). Ex: anthropic/claude-opus-4.8. Vide = defaut du provider." + default: "" passes: description: "Passes self-consistency (>1 ne retient que les G3 stables)" default: "1" @@ -38,6 +46,8 @@ jobs: CORPUS: AgentPublic/eval-stt-officiels RESULTS: AgentPublic/eval-stt-results MODE: ${{ inputs.mode }} + JUDGE_PROVIDER: ${{ inputs.judge_provider }} + JUDGE_MODEL: ${{ inputs.judge_model }} PASSES: ${{ inputs.passes }} steps: - uses: actions/checkout@v4 @@ -66,14 +76,24 @@ jobs: mkdir -p results/reports REV="$(uvx --from huggingface_hub python -c "from huggingface_hub import HfApi; print(HfApi().dataset_info('$CORPUS').sha)")" STAMP="$(date -u +%Y-%m-%dT%H-%M-%SZ)" - REPORT="results/reports/${STAMP}__gravite-${MODE}.md" + # Slug = mode (+ modele juge si override) pour ne pas ecraser un autre run + # (ex. gravite-single-anthropic_claude-opus-4.8.md vs gravite-single.md). + SLUG="$MODE" + if [ -n "$JUDGE_MODEL" ]; then + SLUG="${MODE}-$(echo "$JUDGE_MODEL" | tr '/:' '__')" + fi + REPORT="results/reports/${STAMP}__gravite-${SLUG}.md" if [ "$MODE" = "panel" ]; then - uv run eval-transcript panel --passes "$PASSES" --output "$REPORT" + if [ -n "$JUDGE_MODEL" ]; then + uv run eval-transcript panel --judge albert --judge "openrouter:$JUDGE_MODEL" --passes "$PASSES" --output "$REPORT" + else + uv run eval-transcript panel --passes "$PASSES" --output "$REPORT" + fi else - uv run eval-transcript judge --judge-provider albert --passes "$PASSES" --output "$REPORT" + uv run eval-transcript judge --judge-provider "$JUDGE_PROVIDER" ${JUDGE_MODEL:+--judge-model "$JUDGE_MODEL"} --passes "$PASSES" --output "$REPORT" fi # Estampille la revision du corpus en tete de rapport (reproductibilite) - sed -i "1i " "$REPORT" + sed -i "1i " "$REPORT" cat "$REPORT" - name: Rapport vers HF From 9a02b51a005f1d42098a272730f547807b4fa96d Mon Sep 17 00:00:00 2001 From: benoitvx Date: Wed, 24 Jun 2026 14:18:05 +0200 Subject: [PATCH 2/2] =?UTF-8?q?feat(ci):=20judge.yml=20panel=20=E2=80=94?= =?UTF-8?q?=20liste=20de=20juges=20+=20seuil=20de=20consensus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ajoute inputs `judges` (CSV PROVIDER[:MODEL], 2..N juges) et `consensus_min` (nb min de juges confirmant un G3). Permet un vrai consensus N-juges en CI : panel --judge ... --judge ... --judge ... --mode consensus --consensus-min K. Rétro-compatible : panel sans `judges` garde l'ancien comportement. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/judge.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/judge.yml b/.github/workflows/judge.yml index c75a2f6..f8d8125 100644 --- a/.github/workflows/judge.yml +++ b/.github/workflows/judge.yml @@ -24,6 +24,12 @@ on: judge_model: description: "Modele juge (override). Ex: anthropic/claude-opus-4.8. Vide = defaut du provider." default: "" + judges: + description: "panel uniquement : liste CSV de juges PROVIDER[:MODEL]. Ex: albert:mistral-medium-2508,albert:openai/gpt-oss-120b,openrouter:anthropic/claude-opus-4.8. Vide = defaut (albert + openrouter)." + default: "" + consensus_min: + description: "panel uniquement : nb min de juges qui doivent confirmer un G3. Vide = majorite stricte." + default: "" passes: description: "Passes self-consistency (>1 ne retient que les G3 stables)" default: "1" @@ -48,6 +54,8 @@ jobs: MODE: ${{ inputs.mode }} JUDGE_PROVIDER: ${{ inputs.judge_provider }} JUDGE_MODEL: ${{ inputs.judge_model }} + JUDGES_CSV: ${{ inputs.judges }} + CONSENSUS_MIN: ${{ inputs.consensus_min }} PASSES: ${{ inputs.passes }} steps: - uses: actions/checkout@v4 @@ -79,21 +87,27 @@ jobs: # Slug = mode (+ modele juge si override) pour ne pas ecraser un autre run # (ex. gravite-single-anthropic_claude-opus-4.8.md vs gravite-single.md). SLUG="$MODE" - if [ -n "$JUDGE_MODEL" ]; then + if [ "$MODE" = "single" ] && [ -n "$JUDGE_MODEL" ]; then SLUG="${MODE}-$(echo "$JUDGE_MODEL" | tr '/:' '__')" fi + if [ "$MODE" = "panel" ] && [ -n "$JUDGES_CSV" ]; then + SLUG="panel-consensus${CONSENSUS_MIN:+-min$CONSENSUS_MIN}" + fi REPORT="results/reports/${STAMP}__gravite-${SLUG}.md" if [ "$MODE" = "panel" ]; then - if [ -n "$JUDGE_MODEL" ]; then - uv run eval-transcript panel --judge albert --judge "openrouter:$JUDGE_MODEL" --passes "$PASSES" --output "$REPORT" - else - uv run eval-transcript panel --passes "$PASSES" --output "$REPORT" + JUDGE_ARGS="" + if [ -n "$JUDGES_CSV" ]; then + IFS=',' read -ra JS <<< "$JUDGES_CSV" + for j in "${JS[@]}"; do JUDGE_ARGS="$JUDGE_ARGS --judge $j"; done + elif [ -n "$JUDGE_MODEL" ]; then + JUDGE_ARGS="--judge albert --judge openrouter:$JUDGE_MODEL" fi + uv run eval-transcript panel $JUDGE_ARGS --mode consensus ${CONSENSUS_MIN:+--consensus-min "$CONSENSUS_MIN"} --passes "$PASSES" --output "$REPORT" else uv run eval-transcript judge --judge-provider "$JUDGE_PROVIDER" ${JUDGE_MODEL:+--judge-model "$JUDGE_MODEL"} --passes "$PASSES" --output "$REPORT" fi # Estampille la revision du corpus en tete de rapport (reproductibilite) - sed -i "1i " "$REPORT" + sed -i "1i " "$REPORT" cat "$REPORT" - name: Rapport vers HF