diff --git a/.github/workflows/judge.yml b/.github/workflows/judge.yml index 98855dd..f8d8125 100644 --- a/.github/workflows/judge.yml +++ b/.github/workflows/judge.yml @@ -12,10 +12,24 @@ on: workflow_dispatch: inputs: mode: - description: "single (un juge, Albert/Mistral) ou panel (multi-juges + consensus)" + description: "single (un juge) ou panel (multi-juges + consensus)" type: choice options: [single, panel] default: single + judge_provider: + description: "single uniquement : fournisseur du juge" + type: choice + options: [albert, openrouter] + default: albert + judge_model: + description: "Modele juge (override). Ex: anthropic/claude-opus-4.8. Vide = defaut du provider." + default: "" + judges: + description: "panel uniquement : liste CSV de juges PROVIDER[:MODEL]. Ex: albert:mistral-medium-2508,albert:openai/gpt-oss-120b,openrouter:anthropic/claude-opus-4.8. Vide = defaut (albert + openrouter)." + default: "" + consensus_min: + description: "panel uniquement : nb min de juges qui doivent confirmer un G3. Vide = majorite stricte." + default: "" passes: description: "Passes self-consistency (>1 ne retient que les G3 stables)" default: "1" @@ -38,6 +52,10 @@ jobs: CORPUS: AgentPublic/eval-stt-officiels RESULTS: AgentPublic/eval-stt-results MODE: ${{ inputs.mode }} + JUDGE_PROVIDER: ${{ inputs.judge_provider }} + JUDGE_MODEL: ${{ inputs.judge_model }} + JUDGES_CSV: ${{ inputs.judges }} + CONSENSUS_MIN: ${{ inputs.consensus_min }} PASSES: ${{ inputs.passes }} steps: - uses: actions/checkout@v4 @@ -66,14 +84,30 @@ jobs: mkdir -p results/reports REV="$(uvx --from huggingface_hub python -c "from huggingface_hub import HfApi; print(HfApi().dataset_info('$CORPUS').sha)")" STAMP="$(date -u +%Y-%m-%dT%H-%M-%SZ)" - REPORT="results/reports/${STAMP}__gravite-${MODE}.md" + # Slug = mode (+ modele juge si override) pour ne pas ecraser un autre run + # (ex. gravite-single-anthropic_claude-opus-4.8.md vs gravite-single.md). + SLUG="$MODE" + if [ "$MODE" = "single" ] && [ -n "$JUDGE_MODEL" ]; then + SLUG="${MODE}-$(echo "$JUDGE_MODEL" | tr '/:' '__')" + fi + if [ "$MODE" = "panel" ] && [ -n "$JUDGES_CSV" ]; then + SLUG="panel-consensus${CONSENSUS_MIN:+-min$CONSENSUS_MIN}" + fi + REPORT="results/reports/${STAMP}__gravite-${SLUG}.md" if [ "$MODE" = "panel" ]; then - uv run eval-transcript panel --passes "$PASSES" --output "$REPORT" + JUDGE_ARGS="" + if [ -n "$JUDGES_CSV" ]; then + IFS=',' read -ra JS <<< "$JUDGES_CSV" + for j in "${JS[@]}"; do JUDGE_ARGS="$JUDGE_ARGS --judge $j"; done + elif [ -n "$JUDGE_MODEL" ]; then + JUDGE_ARGS="--judge albert --judge openrouter:$JUDGE_MODEL" + fi + uv run eval-transcript panel $JUDGE_ARGS --mode consensus ${CONSENSUS_MIN:+--consensus-min "$CONSENSUS_MIN"} --passes "$PASSES" --output "$REPORT" else - uv run eval-transcript judge --judge-provider albert --passes "$PASSES" --output "$REPORT" + uv run eval-transcript judge --judge-provider "$JUDGE_PROVIDER" ${JUDGE_MODEL:+--judge-model "$JUDGE_MODEL"} --passes "$PASSES" --output "$REPORT" fi # Estampille la revision du corpus en tete de rapport (reproductibilite) - sed -i "1i " "$REPORT" + sed -i "1i " "$REPORT" cat "$REPORT" - name: Rapport vers HF