feat(hle): add Humanity's Last Exam dataset + 0-shot LLM-judge task

pyproject.toml

@@ -29,6 +29,9 @@ Changelog = "https://github.com/scitix/sieval/blob/main/CHANGELOG.md"
 
 [project.optional-dependencies]
 drop = ["numpy<=2.2", "scipy>=1.16.3"]
+# numpy backs the vendored HLE `calib_err` (calibration error). No grader/judge
+# extras: the judge is reached through the base `openai` client like any model.
+hle = ["numpy<=2.2"]
 ifbench = [
   "emoji>=2.15.0",
   "nltk>=3.9.2",

sieval/community/hle.py

@@ -0,0 +1,159 @@
+# Sources (Humanity's Last Exam, Center for AI Safety), pinned to commit
+# 26dca2e253b405105b4c3d8c2f5af06f86f90c66:
+#   - SYSTEM_PROMPT (verbatim) and the user-message assembly this task mirrors:
+#     hle_eval/run_model_predictions.py (SYSTEM_PROMPT, format_message)
+#   - JUDGE_PROMPT (verbatim), the ExtractedAnswer field set, and the accuracy /
+#     confidence-interval / calibration-error formulas: hle_eval/run_judge_results.py
+#     (JUDGE_PROMPT, ExtractedAnswer, dump_metrics)
+#   - calib_err (verbatim): hle_eval/run_judge_results.py, itself vendored from
+#     https://github.com/hendrycks/outlier-exposure/blob/master/utils/calibration_tools.py
+"""Humanity's Last Exam (HLE) prompt + grading assets.
+
+HLE (Center for AI Safety) is a ~2,500-question, multi-domain, closed-ended
+academic benchmark graded by an LLM judge. This module vendors the upstream
+model system prompt, the judge prompt, and the metric kernel (accuracy, 95%
+Wald confidence interval, and calibration error).
+
+Deviations from upstream (``hle_eval/run_judge_results.py`` @ 26dca2e):
+
+* Judge invocation. Upstream calls ``client.beta.chat.completions.parse`` with
+  the ``ExtractedAnswer`` Pydantic ``response_format`` (server-enforced
+  structured output). sieval reaches the judge through the generic ``ChatModel``
+  (``chat.completions`` returning text), so ``parse_judge`` reads the judge's
+  reply textually for the ``correct`` (yes/no) and ``confidence`` fields the
+  prompt already specifies. Why: the judge runs on an arbitrary ``api_base`` and
+  not every endpoint enforces a JSON schema — parsing the prompted field format
+  keeps the judge endpoint-agnostic. Correctness follows upstream semantics
+  (``"yes" in judge_response["correct"]``); an unparseable reply is scored
+  incorrect with confidence 100 (upstream's "no confidence score" default).
+* ``aggregate_metrics`` returns a dict instead of printing, and guards
+  ``len(confidence) < beta``: the vendored ``calib_err`` indexes ``bins[-1]`` and
+  raises ``IndexError`` below ``beta`` judged records — unreachable for the full
+  HLE test set (~2.5k) but reachable via slices/tests, where it reports a
+  calibration error of 0.0. Numerator (correct count) and calibration arrays are
+  built only from judged attempts while the denominator ``n`` is supplied by the
+  caller (finals + failures), mirroring upstream where unjudged/failed
+  predictions stay in ``n`` but out of the ``correct`` array.
+
+AI-Generated Code - Claude Opus 4.8 (Anthropic)
+"""
+
+import math
+import re
+
+import numpy as np
+
+SYSTEM_PROMPT = "Your response should be in the following format:\nExplanation: {your explanation for your answer choice}\nAnswer: {your chosen answer}\nConfidence: {your confidence score between 0% and 100% for your answer}"
+
+JUDGE_PROMPT = r"""Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below.
+
+[question]: {question}
+
+[response]: {response}
+
+Your judgement must be in the format and criteria specified below:
+
+extracted_final_answer: The final exact answer extracted from the [response]. Put the extracted answer as 'None' if there is no exact, final answer to extract from the response.
+
+[correct_answer]: {correct_answer}
+
+reasoning: Explain why the extracted_final_answer is correct or incorrect based on [correct_answer], focusing only on if there are meaningful differences between [correct_answer] and the extracted_final_answer. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match.
+
+correct: Answer 'yes' if extracted_final_answer matches the [correct_answer] given above, or is within a small margin of error for numerical problems. Answer 'no' otherwise, i.e. if there if there is any inconsistency, ambiguity, non-equivalency, or if the extracted answer is incorrect.
+
+
+confidence: The extracted confidence score between 0|\%| and 100|\%| from [response]. Put 100 if there is no confidence score available."""
+
+# Target bin size for calibration error (upstream default).
+BETA = 100
+
+_CORRECT_RE = re.compile(r"correct\s*:\s*\**\s*(yes|no)", re.IGNORECASE)
+_CONFIDENCE_RE = re.compile(r"confidence\s*:\s*\**\s*(\d+)", re.IGNORECASE)
+
+
+def parse_judge(reply: str) -> tuple[bool, int]:
+    """Extract ``(correct, confidence)`` from a judge reply.
+
+    The judge is prompted (see ``JUDGE_PROMPT``) to emit ``correct: yes|no`` and
+    ``confidence: <int>`` fields. Both appear after the free-form ``reasoning``
+    field, so the last match of each pattern wins. Follows upstream semantics
+    (``"yes" in judge_response["correct"]``); an unrecognizable reply is scored
+    incorrect with confidence 100 (upstream's "no confidence score" default).
+    """
+    correct_matches = _CORRECT_RE.findall(reply)
+    correct = bool(correct_matches) and correct_matches[-1].lower() == "yes"
+
+    confidence_matches = _CONFIDENCE_RE.findall(reply)
+    confidence = int(confidence_matches[-1]) if confidence_matches else 100
+    return correct, confidence
+
+
+# source: https://github.com/hendrycks/outlier-exposure/blob/master/utils/calibration_tools.py
+def calib_err(confidence, correct, p="2", beta=100):
+    # beta is target bin size
+    idxs = np.argsort(confidence)
+    confidence = confidence[idxs]
+    correct = correct[idxs]
+    bins = [[i * beta, (i + 1) * beta] for i in range(len(confidence) // beta)]
+    bins[-1] = [bins[-1][0], len(confidence)]
+
+    cerr = 0
+    total_examples = len(confidence)
+    for i in range(len(bins) - 1):
+        bin_confidence = confidence[bins[i][0] : bins[i][1]]
+        bin_correct = correct[bins[i][0] : bins[i][1]]
+        num_examples_in_bin = len(bin_confidence)
+
+        if num_examples_in_bin > 0:
+            difference = np.abs(np.nanmean(bin_confidence) - np.nanmean(bin_correct))
+
+            if p == "2":
+                cerr += num_examples_in_bin / total_examples * np.square(difference)
+            elif p == "1":
+                cerr += num_examples_in_bin / total_examples * difference
+            elif p == "infty" or p == "infinity" or p == "max":
+                cerr = np.maximum(cerr, difference)
+            else:
+                assert False, "p must be '1', '2', or 'infty'"
+
+    if p == "2":
+        cerr = np.sqrt(cerr)
+
+    return cerr
+
+
+def aggregate_metrics(
+    correct: list[bool], confidence: list[int], n: int
+) -> dict[str, float]:
+    """Aggregate judged ``correct``/``confidence`` into HLE metrics.
+
+    Mirrors upstream ``dump_metrics``: ``correct``/``confidence`` are the judged
+    attempts (numerator + calibration arrays), while ``n`` is the full requested
+    count (denominator) so failed/ungraded attempts count as incorrect.
+
+    Returns accuracy (%), the 95% Wald half-width (``confidence_interval``, in
+    percentage points), and ``calibration_error`` (0..100). ``n == 0`` yields
+    all-zero metrics; fewer than ``BETA`` judged attempts yields
+    ``calibration_error`` 0.0 (see module docstring).
+    """
+    if n == 0:
+        return {"accuracy": 0.0, "confidence_interval": 0.0, "calibration_error": 0.0}
+
+    accuracy = round(100 * sum(correct) / n, 2)
+    # Wald estimator, 95% confidence interval
+    confidence_interval = round(1.96 * math.sqrt(accuracy * (100 - accuracy) / n), 2)
+
+    if len(confidence) >= BETA:
+        confidence_arr = np.array(confidence) / 100
+        correct_arr = np.array(correct)
+        calibration_error = 100 * round(
+            float(calib_err(confidence_arr, correct_arr, p="2", beta=BETA)), 2
+        )
+    else:
+        calibration_error = 0.0
+
+    return {
+        "accuracy": accuracy,
+        "confidence_interval": confidence_interval,
+        "calibration_error": calibration_error,
+    }

sieval/datasets/__init__.pyi

@@ -53,6 +53,10 @@ from .hendrycks_math import (
     HendrycksMathDataset,
     HendrycksMathDatasetSample,
 )
+from .hle import (
+    HLEDataset,
+    HLEDatasetSample,
+)
 from .hmmt_feb_2025 import (
     HMMTFeb2025Dataset,
     HMMTFeb2025DatasetSample,
@@ -141,6 +145,8 @@ __all__ = [
     "GPQADiamondDatasetSample",
     "GSM8KDataset",
     "GSM8KDatasetSample",
+    "HLEDataset",
+    "HLEDatasetSample",
     "HMMTFeb2025Dataset",
     "HMMTFeb2025DatasetSample",
     "HMMTFeb2026Dataset",

sieval/datasets/hle.py

@@ -0,0 +1,69 @@
+"""Humanity's Last Exam (HLE) dataset loader (Center for AI Safety).
+
+HLE (Phan et al., 2025) is a multi-domain, closed-ended academic benchmark of
+frontier-difficulty questions (mathematics, sciences, humanities), each with an
+``exactMatch`` or ``multipleChoice`` gold answer suitable for automated
+LLM-judge grading. The Hub repo exposes a single ``test`` split; this loader
+mirrors it as-is.
+
+The model-facing image is the ``image`` column — a plain string (a base64 data
+URI, ``""`` when absent), preserved untouched. The repo also ships two auxiliary
+``Image`` feature columns (``image_preview``, ``rationale_image``) that default
+to ``decode=True``; no task consumes them, but materializing a row would decode
+them and pull in Pillow. Decoding is disabled here (``Image(decode=False)``) so
+the rows stay Pillow-free while keeping the raw bytes available; text-only vs.
+full-set selection remains the task's concern.
+
+AI-Generated Code - Claude Opus 4.8 (Anthropic)
+"""
+
+from typing import TypedDict, override
+
+from datasets import DatasetDict as HFDatasetDict
+from datasets import Image, load_dataset
+
+from sieval.core.datasets import (
+    Category,
+    Dataset,
+    Level1Category,
+    sieval_dataset,
+)
+from sieval.core.utils.hf import ensure_dataset_dict
+
+# Pin the Hub revision for reproducibility (`main` at integration time).
+HLE_REVISION = "5a81a4c7271a2a2a312b9a690f0c2fde837e4c29"
+
+
+class HLEDatasetSample(TypedDict):
+    id: str
+    question: str
+    image: str
+    answer: str
+    answer_type: str
+    author_name: str
+    rationale: str
+    raw_subject: str
+    category: str
+
+
+@sieval_dataset(
+    name="hle",
+    display_name="Humanity's Last Exam",
+    description="HLE — multi-domain, closed-ended frontier academic benchmark.",
+    source=f"hf:cais/hle@{HLE_REVISION}",
+    categories=(Category(Level1Category.KNOWLEDGE, "Multi-domain"),),
+    tags=("english", "reasoning", "academic"),
+    license="MIT",
+)
+class HLEDataset(Dataset[HLEDatasetSample]):
+    # Auxiliary Image feature columns no task consumes; decoding them on row
+    # access would require Pillow, so disable it (raw bytes are kept).
+    _IMAGE_FEATURE_COLUMNS = ("image_preview", "rationale_image")
+
+    @override
+    def load(self, name_or_path: str, **kwargs) -> HFDatasetDict:
+        dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs))
+        for split in dataset:
+            for column in self._IMAGE_FEATURE_COLUMNS:
+                dataset[split] = dataset[split].cast_column(column, Image(decode=False))
+        return dataset

sieval/meta/index.json

@@ -295,6 +295,28 @@
       "license": "MIT",
       "checksums": {}
     },
+    {
+      "name": "hle",
+      "display_name": "Humanity's Last Exam",
+      "description": "HLE — multi-domain, closed-ended frontier academic benchmark.",
+      "source": [
+        "hf:cais/hle@5a81a4c7271a2a2a312b9a690f0c2fde837e4c29"
+      ],
+      "categories": [
+        {
+          "level1": "Knowledge",
+          "level2": "Multi-domain"
+        }
+      ],
+      "tags": [
+        "english",
+        "reasoning",
+        "academic"
+      ],
+      "deps_group": null,
+      "license": "MIT",
+      "checksums": {}
+    },
     {
       "name": "hmmt_feb_2025",
       "display_name": "HMMT Feb 2025",
@@ -983,6 +1005,27 @@
       },
       "status": "stable"
     },
+    {
+      "name": "hle_0shot_gen",
+      "display_name": "Humanity's Last Exam (0-shot, generative)",
+      "description": "Multi-domain frontier academic QA graded by an LLM judge.",
+      "dataset": "hle",
+      "eval_mode": "gen",
+      "n_shot": 0,
+      "tags": [
+        "english",
+        "reasoning",
+        "academic"
+      ],
+      "deps_group": "hle",
+      "model_type": "chat",
+      "reference_impl": {
+        "source": "hle",
+        "url": "https://github.com/centerforaisafety/hle/tree/26dca2e253b405105b4c3d8c2f5af06f86f90c66/hle_eval",
+        "notes": "Generative port of Humanity's Last Exam (Center for AI Safety). SYSTEM_PROMPT and JUDGE_PROMPT are vendored byte-for-byte; the judge runs through sieval's ChatModel (text) rather than upstream's beta.chat.completions.parse structured output, and its correct/confidence fields are parsed from the reply. Metrics mirror upstream dump_metrics: accuracy, a 95% Wald confidence interval, and calibration error (calib_err, p=2, beta=100). Text-only subset by default (text_only=True); full set (text_only=False) is marked * in reports. Grader is a REAL LLM (upstream default o3-mini-2025-01-31) supplied via the `grader` task arg on its own api_base/api_key. REPRODUCIBILITY: scores depend on the judge endpoint's model version (not pinnable like a Hub revision) — pin the grader model; the per-sample correct/confidence and grader model id are persisted."
+      },
+      "status": "stable"
+    },
     {
       "name": "hmmt_feb_2025_0shot_gen",
       "display_name": "HMMT Feb 2025 (0-shot, generative)",

sieval/tasks/__init__.pyi

@@ -49,6 +49,9 @@ from .hellaswag_kshot_ppl import (
 from .hendrycks_math_kshot_base_gen import (
     HendrycksMathFewShotBaseGenTask,
 )
+from .hle_0shot_gen import (
+    HLEZeroShotGenTask,
+)
 from .hmmt_feb_2025_0shot_gen import (
     HMMTFeb2025ZeroShotGenTask,
 )
@@ -122,6 +125,7 @@ __all__ = [
     "GPQADiamondZeroShotGenTask",
     "GSM8KFewShotBaseGenTask",
     "GSM8KZeroShotGenTask",
+    "HLEZeroShotGenTask",
     "HMMTFeb2025ZeroShotGenTask",
     "HMMTFeb2026ZeroShotGenTask",
     "HellaSwagFewShotPPLTask",