diff --git a/pdm.lock b/pdm.lock index 4002a16..264c777 100644 --- a/pdm.lock +++ b/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "dev", "drop", "ifbench", "ifeval", "math", "t-eval", "test"] +groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "math", "t-eval", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:aadbe110ed98781ee5c51ae38bd40780304ed26da3c371a0fd3f5ff1fd1256ef" +content_hash = "sha256:17643547b4cfe3997a8b1cceb3b96679566ea4601d23e8c49daba38473ac819f" [[metadata.targets]] requires_python = ">=3.12,<3.15" @@ -1488,7 +1488,7 @@ name = "numpy" version = "2.2.0" requires_python = ">=3.10" summary = "Fundamental package for array computing in Python" -groups = ["default", "drop", "t-eval"] +groups = ["default", "drop", "hle", "t-eval"] files = [ {file = "numpy-2.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cff210198bb4cae3f3c100444c5eaa573a823f05c253e7188e1362a5555235b3"}, {file = "numpy-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b92a5828bd4d9aa0952492b7de803135038de47343b2aa3cc23f3b71a3dc4e"}, diff --git a/pyproject.toml b/pyproject.toml index 4de8eb8..f94472e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,9 @@ Changelog = "https://github.com/scitix/sieval/blob/main/CHANGELOG.md" [project.optional-dependencies] drop = ["numpy<=2.2", "scipy>=1.16.3"] +# numpy backs the vendored HLE `calib_err` (calibration error). No grader/judge +# extras: the judge is reached through the base `openai` client like any model. +hle = ["numpy<=2.2"] ifbench = [ "emoji>=2.15.0", "nltk>=3.9.2", diff --git a/sieval/community/hle.py b/sieval/community/hle.py new file mode 100644 index 0000000..4f05106 --- /dev/null +++ b/sieval/community/hle.py @@ -0,0 +1,160 @@ +# Sources (Humanity's Last Exam, Center for AI Safety), pinned to commit +# 26dca2e253b405105b4c3d8c2f5af06f86f90c66: +# - SYSTEM_PROMPT (verbatim) and the user-message assembly this task mirrors: +# hle_eval/run_model_predictions.py (SYSTEM_PROMPT, format_message) +# - JUDGE_PROMPT (verbatim), the ExtractedAnswer field set, and the accuracy / +# confidence-interval / calibration-error formulas: hle_eval/run_judge_results.py +# (JUDGE_PROMPT, ExtractedAnswer, dump_metrics) +# - calib_err (verbatim): hle_eval/run_judge_results.py, itself vendored from +# https://github.com/hendrycks/outlier-exposure/blob/master/utils/calibration_tools.py +"""Humanity's Last Exam (HLE) prompt + grading assets. + +HLE (Center for AI Safety) is a ~2,500-question, multi-domain, closed-ended +academic benchmark graded by an LLM judge. This module vendors the upstream +model system prompt, the judge prompt, and the metric kernel (accuracy, 95% +Wald confidence interval, and calibration error). + +Deviations from upstream (``hle_eval/run_judge_results.py`` @ 26dca2e): + +* Judge invocation. Upstream calls ``client.beta.chat.completions.parse`` with + the ``ExtractedAnswer`` Pydantic ``response_format`` (server-enforced + structured output). sieval reaches the judge through the generic ``ChatModel`` + (``chat.completions`` returning text), so ``parse_judge`` reads the judge's + reply textually for the ``correct`` (yes/no) and ``confidence`` fields the + prompt already specifies. Why: the judge runs on an arbitrary ``api_base`` and + not every endpoint enforces a JSON schema — parsing the prompted field format + keeps the judge endpoint-agnostic. Correctness follows upstream semantics + (``"yes" in judge_response["correct"]``); an unparseable reply is scored + incorrect with confidence 100 (upstream's "no confidence score" default). +* ``aggregate_metrics`` returns a dict instead of printing, and guards + ``len(confidence) < beta``: the vendored ``calib_err`` indexes ``bins[-1]`` and + raises ``IndexError`` below ``beta`` judged records — unreachable for the full + HLE test set (~2.5k) but reachable via slices/tests, where it reports a + calibration error of 0.0. Numerator (correct count) and calibration arrays are + built only from judged attempts while the denominator ``n`` is supplied by the + caller (finals + failures), mirroring upstream where unjudged/failed + predictions stay in ``n`` but out of the ``correct`` array. + +AI-Generated Code - Claude Opus 4.8 (Anthropic) +""" + +import math +import re + +import numpy as np + +SYSTEM_PROMPT = "Your response should be in the following format:\nExplanation: {your explanation for your answer choice}\nAnswer: {your chosen answer}\nConfidence: {your confidence score between 0% and 100% for your answer}" + +JUDGE_PROMPT = r"""Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below. + +[question]: {question} + +[response]: {response} + +Your judgement must be in the format and criteria specified below: + +extracted_final_answer: The final exact answer extracted from the [response]. Put the extracted answer as 'None' if there is no exact, final answer to extract from the response. + +[correct_answer]: {correct_answer} + +reasoning: Explain why the extracted_final_answer is correct or incorrect based on [correct_answer], focusing only on if there are meaningful differences between [correct_answer] and the extracted_final_answer. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match. + +correct: Answer 'yes' if extracted_final_answer matches the [correct_answer] given above, or is within a small margin of error for numerical problems. Answer 'no' otherwise, i.e. if there if there is any inconsistency, ambiguity, non-equivalency, or if the extracted answer is incorrect. + + +confidence: The extracted confidence score between 0|\%| and 100|\%| from [response]. Put 100 if there is no confidence score available.""" + +# Target bin size for calibration error (upstream default). +BETA = 100 + +# `\b` anchors the field name so it never matches inside "incorrect:". +_CORRECT_RE = re.compile(r"\bcorrect\s*:\s*\**\s*(yes|no)", re.IGNORECASE) +_CONFIDENCE_RE = re.compile(r"confidence\s*:\s*\**\s*(\d+)", re.IGNORECASE) + + +def parse_judge(reply: str) -> tuple[bool, int]: + """Extract ``(correct, confidence)`` from a judge reply. + + The judge is prompted (see ``JUDGE_PROMPT``) to emit ``correct: yes|no`` and + ``confidence: `` fields. Both appear after the free-form ``reasoning`` + field, so the last match of each pattern wins. Follows upstream semantics + (``"yes" in judge_response["correct"]``); an unrecognizable reply is scored + incorrect with confidence 100 (upstream's "no confidence score" default). + """ + correct_matches = _CORRECT_RE.findall(reply) + correct = bool(correct_matches) and correct_matches[-1].lower() == "yes" + + confidence_matches = _CONFIDENCE_RE.findall(reply) + confidence = int(confidence_matches[-1]) if confidence_matches else 100 + return correct, confidence + + +# source: https://github.com/hendrycks/outlier-exposure/blob/master/utils/calibration_tools.py +def calib_err(confidence, correct, p="2", beta=100): + # beta is target bin size + idxs = np.argsort(confidence) + confidence = confidence[idxs] + correct = correct[idxs] + bins = [[i * beta, (i + 1) * beta] for i in range(len(confidence) // beta)] + bins[-1] = [bins[-1][0], len(confidence)] + + cerr = 0 + total_examples = len(confidence) + for i in range(len(bins) - 1): + bin_confidence = confidence[bins[i][0] : bins[i][1]] + bin_correct = correct[bins[i][0] : bins[i][1]] + num_examples_in_bin = len(bin_confidence) + + if num_examples_in_bin > 0: + difference = np.abs(np.nanmean(bin_confidence) - np.nanmean(bin_correct)) + + if p == "2": + cerr += num_examples_in_bin / total_examples * np.square(difference) + elif p == "1": + cerr += num_examples_in_bin / total_examples * difference + elif p == "infty" or p == "infinity" or p == "max": + cerr = np.maximum(cerr, difference) + else: + assert False, "p must be '1', '2', or 'infty'" + + if p == "2": + cerr = np.sqrt(cerr) + + return cerr + + +def aggregate_metrics( + correct: list[bool], confidence: list[int], n: int +) -> dict[str, float]: + """Aggregate judged ``correct``/``confidence`` into HLE metrics. + + Mirrors upstream ``dump_metrics``: ``correct``/``confidence`` are the judged + attempts (numerator + calibration arrays), while ``n`` is the full requested + count (denominator) so failed/ungraded attempts count as incorrect. + + Returns accuracy (%), the 95% Wald half-width (``confidence_interval``, in + percentage points), and ``calibration_error`` (0..100). ``n == 0`` yields + all-zero metrics; fewer than ``BETA`` judged attempts yields + ``calibration_error`` 0.0 (see module docstring). + """ + if n == 0: + return {"accuracy": 0.0, "confidence_interval": 0.0, "calibration_error": 0.0} + + accuracy = round(100 * sum(correct) / n, 2) + # Wald estimator, 95% confidence interval + confidence_interval = round(1.96 * math.sqrt(accuracy * (100 - accuracy) / n), 2) + + if len(confidence) >= BETA: + confidence_arr = np.array(confidence) / 100 + correct_arr = np.array(correct) + calibration_error = 100 * round( + float(calib_err(confidence_arr, correct_arr, p="2", beta=BETA)), 2 + ) + else: + calibration_error = 0.0 + + return { + "accuracy": accuracy, + "confidence_interval": confidence_interval, + "calibration_error": calibration_error, + } diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index 1973cbc..c1f2bb9 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -53,6 +53,10 @@ from .hendrycks_math import ( HendrycksMathDataset, HendrycksMathDatasetSample, ) +from .hle import ( + HLEDataset, + HLEDatasetSample, +) from .hmmt_feb_2025 import ( HMMTFeb2025Dataset, HMMTFeb2025DatasetSample, @@ -141,6 +145,8 @@ __all__ = [ "GPQADiamondDatasetSample", "GSM8KDataset", "GSM8KDatasetSample", + "HLEDataset", + "HLEDatasetSample", "HMMTFeb2025Dataset", "HMMTFeb2025DatasetSample", "HMMTFeb2026Dataset", diff --git a/sieval/datasets/hle.py b/sieval/datasets/hle.py new file mode 100644 index 0000000..6f53757 --- /dev/null +++ b/sieval/datasets/hle.py @@ -0,0 +1,69 @@ +"""Humanity's Last Exam (HLE) dataset loader (Center for AI Safety). + +HLE (Phan et al., 2025) is a multi-domain, closed-ended academic benchmark of +frontier-difficulty questions (mathematics, sciences, humanities), each with an +``exactMatch`` or ``multipleChoice`` gold answer suitable for automated +LLM-judge grading. The Hub repo exposes a single ``test`` split; this loader +mirrors it as-is. + +The model-facing image is the ``image`` column — a plain string (a base64 data +URI, ``""`` when absent), preserved untouched. The repo also ships two auxiliary +``Image`` feature columns (``image_preview``, ``rationale_image``) that default +to ``decode=True``; no task consumes them, but materializing a row would decode +them and pull in Pillow. Decoding is disabled here (``Image(decode=False)``) so +the rows stay Pillow-free while keeping the raw bytes available; text-only vs. +full-set selection remains the task's concern. + +AI-Generated Code - Claude Opus 4.8 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import Image, load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +# Pin the Hub revision for reproducibility (`main` at integration time). +HLE_REVISION = "5a81a4c7271a2a2a312b9a690f0c2fde837e4c29" + + +class HLEDatasetSample(TypedDict): + id: str + question: str + image: str + answer: str + answer_type: str + author_name: str + rationale: str + raw_subject: str + category: str + + +@sieval_dataset( + name="hle", + display_name="Humanity's Last Exam", + description="HLE — multi-domain, closed-ended frontier academic benchmark.", + source=f"hf:cais/hle@{HLE_REVISION}", + categories=(Category(Level1Category.KNOWLEDGE, "Multi-domain"),), + tags=("english", "reasoning", "academic"), + license="MIT", +) +class HLEDataset(Dataset[HLEDatasetSample]): + # Auxiliary Image feature columns no task consumes; decoding them on row + # access would require Pillow, so disable it (raw bytes are kept). + _IMAGE_FEATURE_COLUMNS = ("image_preview", "rationale_image") + + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs)) + for split in dataset: + for column in self._IMAGE_FEATURE_COLUMNS: + dataset[split] = dataset[split].cast_column(column, Image(decode=False)) + return dataset diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 1df783e..6583539 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -295,6 +295,28 @@ "license": "MIT", "checksums": {} }, + { + "name": "hle", + "display_name": "Humanity's Last Exam", + "description": "HLE — multi-domain, closed-ended frontier academic benchmark.", + "source": [ + "hf:cais/hle@5a81a4c7271a2a2a312b9a690f0c2fde837e4c29" + ], + "categories": [ + { + "level1": "Knowledge", + "level2": "Multi-domain" + } + ], + "tags": [ + "english", + "reasoning", + "academic" + ], + "deps_group": null, + "license": "MIT", + "checksums": {} + }, { "name": "hmmt_feb_2025", "display_name": "HMMT Feb 2025", @@ -983,6 +1005,27 @@ }, "status": "stable" }, + { + "name": "hle_0shot_gen", + "display_name": "Humanity's Last Exam (0-shot, generative)", + "description": "Multi-domain frontier academic QA graded by an LLM judge.", + "dataset": "hle", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "reasoning", + "academic" + ], + "deps_group": "hle", + "model_type": "chat", + "reference_impl": { + "source": "hle", + "url": "https://github.com/centerforaisafety/hle/tree/26dca2e253b405105b4c3d8c2f5af06f86f90c66/hle_eval", + "notes": "Generative port of Humanity's Last Exam (Center for AI Safety). SYSTEM_PROMPT and JUDGE_PROMPT are vendored byte-for-byte; the judge runs through sieval's ChatModel (text) rather than upstream's beta.chat.completions.parse structured output, and its correct/confidence fields are parsed from the reply. Metrics mirror upstream dump_metrics: accuracy, a 95% Wald confidence interval, and calibration error (calib_err, p=2, beta=100). Text-only subset by default (text_only=True); full set (text_only=False) is marked * in reports. Grader is a REAL LLM (upstream default o3-mini-2025-01-31) supplied via the `grader` task arg on its own api_base/api_key. REPRODUCIBILITY: scores depend on the judge endpoint's model version (not pinnable like a Hub revision) — pin the grader model; the per-sample correct/confidence and grader model id are persisted. VALIDATION: gpt-oss-20b scored 12.14 / 3.61 (reasoning=high / low, judge GPT-5.2, text-only, no tools) vs the gpt-oss model card (arXiv:2508.10925) 10.9 / 4.2 — within <3pp." + }, + "status": "stable" + }, { "name": "hmmt_feb_2025_0shot_gen", "display_name": "HMMT Feb 2025 (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 96e1cd3..ed888a6 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -49,6 +49,9 @@ from .hellaswag_kshot_ppl import ( from .hendrycks_math_kshot_base_gen import ( HendrycksMathFewShotBaseGenTask, ) +from .hle_0shot_gen import ( + HLEZeroShotGenTask, +) from .hmmt_feb_2025_0shot_gen import ( HMMTFeb2025ZeroShotGenTask, ) @@ -122,6 +125,7 @@ __all__ = [ "GPQADiamondZeroShotGenTask", "GSM8KFewShotBaseGenTask", "GSM8KZeroShotGenTask", + "HLEZeroShotGenTask", "HMMTFeb2025ZeroShotGenTask", "HMMTFeb2026ZeroShotGenTask", "HellaSwagFewShotPPLTask", diff --git a/sieval/tasks/hle_0shot_gen.py b/sieval/tasks/hle_0shot_gen.py new file mode 100644 index 0000000..acd7c02 --- /dev/null +++ b/sieval/tasks/hle_0shot_gen.py @@ -0,0 +1,257 @@ +"""Humanity's Last Exam (HLE) — 0-shot generative, LLM-judge graded. + +Generative port of HLE (Center for AI Safety; Phan et al., 2025). The model +answers a closed-ended, multi-domain academic question under the HLE system +prompt (an ``Explanation / Answer / Confidence`` format), and a separate **LLM +judge** decides whether the free-form answer matches the gold. Headline metric +is accuracy; the judge also extracts the model's confidence, from which a +calibration error is computed (alongside a 95% Wald confidence interval). + +Subset: the **text-only** subset is graded by default (``text_only=True``); +technical reports mark full-set (text + image) numbers with ``*``. Image +questions carry a base64 data URI in the ``image`` column; ``text_only`` drops +them, and with ``text_only=False`` the image is attached as an ``image_url`` +content block (requires a vision-capable candidate + judge). + +Deviations from upstream (``hle_eval`` @ 26dca2e; see ``sieval.community.hle``): + +* The upstream o1-only ``system``→``user`` role swap is dropped; the system + prompt is always sent as ``system`` (correct for general instruct models). +* The judge is reached through ``ChatModel`` (text), not upstream's + ``beta.chat.completions.parse`` structured output; its ``correct``/``confidence`` + fields are parsed from the reply (see ``sieval.community.hle.parse_judge``). +* Calibration error is guarded below the bin size for slices/tests (docs there). + +Decoding params are model-layer, set via ``models:`` / ``infer_args`` — never by +this task. Upstream HLE defaults to ``temperature=0`` and advises +``max_completion_tokens>=8192`` for reasoning models; specific reproductions +override these (e.g. a technical report may evaluate at ``temperature=1.0``, +``top_p=0.95`` with a large token budget). + +Grader is a REAL LLM supplied via the ``grader`` task arg on its own +``api_base``/``api_key``. Correctness depends on the judge endpoint's model +version (not pinnable like a Hub revision) — pin the grader model for +reproducibility; each sample's ``correct``, ``confidence`` and grader model id +are persisted in the feedback record. + +Target: report against technical-report HLE numbers (e.g. the GLM series +evaluates the text-only subset with a strong LLM judge, such as GPT-5.2); the +grading protocol (judge model, subset) is report-specific, so cite it alongside +any comparison. + +AI-Generated Code - Claude Opus 4.8 (Anthropic) +""" + +from collections.abc import Mapping +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from openai.types.chat import ChatCompletionMessageParam + +from sieval.community.hle import ( + JUDGE_PROMPT, + SYSTEM_PROMPT, + aggregate_metrics, + parse_judge, +) +from sieval.core.datasets import Dataset +from sieval.core.models import ChatModel, Model, ModelOutput +from sieval.core.tasks import ( + EvalMode, + ReferenceImpl, + Task, + sieval_task, +) +from sieval.datasets import HLEDatasetSample + + +class JudgeFeedback(TypedDict): + correct: bool + confidence: int + gold: str + predicted: str + grader_model: str + + +@sieval_task( + name="hle_0shot_gen", + display_name="Humanity's Last Exam (0-shot, generative)", + description="Multi-domain frontier academic QA graded by an LLM judge.", + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "reasoning", "academic"), + model_type="chat", + deps_group="hle", + status="stable", + reference_impl=ReferenceImpl( + source="hle", + url="https://github.com/centerforaisafety/hle/tree/26dca2e253b405105b4c3d8c2f5af06f86f90c66/hle_eval", + notes=( + "Generative port of Humanity's Last Exam (Center for AI Safety). " + "SYSTEM_PROMPT and JUDGE_PROMPT are vendored byte-for-byte; the judge " + "runs through sieval's ChatModel (text) rather than upstream's " + "beta.chat.completions.parse structured output, and its correct/" + "confidence fields are parsed from the reply. Metrics mirror upstream " + "dump_metrics: accuracy, a 95% Wald confidence interval, and " + "calibration error (calib_err, p=2, beta=100). Text-only subset by " + "default (text_only=True); full set (text_only=False) is marked * in " + "reports. Grader is a REAL LLM (upstream default o3-mini-2025-01-31) " + "supplied via the `grader` task arg on its own api_base/api_key. " + "REPRODUCIBILITY: scores depend on the judge endpoint's model version " + "(not pinnable like a Hub revision) — pin the grader model; the " + "per-sample correct/confidence and grader model id are persisted. " + "VALIDATION: gpt-oss-20b scored 12.14 / 3.61 (reasoning=high / low, " + "judge GPT-5.2, text-only, no tools) vs the gpt-oss model card " + "(arXiv:2508.10925) 10.9 / 4.2 — within <3pp." + ), + ), +) +class HLEZeroShotGenTask( + Task[ + HLEDatasetSample, + list[ChatCompletionMessageParam], + ModelOutput, + list[str], + list[JudgeFeedback], + dict[str, float], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + grader: Mapping | Model | None = None, + n: int = 1, + text_only: bool = True, + ): + if text_only: + dataset = self._select_text_only(dataset) + super().__init__(dataset=dataset, model=model, name=name) + self._n = n + self._grader = self._build_grader(grader) + + @staticmethod + def _select_text_only( + dataset: Dataset[HLEDatasetSample], + ) -> Dataset[HLEDatasetSample]: + """Return a clone keeping only text-only questions (empty ``image``). + + ``image`` is a string column (``""`` when absent). The filter reads only + that column (``input_columns=["image"]``) so HF never materializes the + sibling ``image_preview`` / ``rationale_image`` ``Image`` features, which + default to ``decode=True`` and would otherwise pull in Pillow. Raises if + the filter empties the ``test`` split — a signal that the ``image`` + column is missing or every question is multi-modal. + """ + source = dataset.dataset_dict + filtered = HFDatasetDict( + { + split: ds.filter(lambda image: not image, input_columns=["image"]) + for split, ds in source.items() + } + ) + if "test" in filtered and len(filtered["test"]) == 0: + raise ValueError( + "HLE text-only selection produced an empty 'test' split; the " + "source may lack the 'image' column or contain only multi-modal " + "questions." + ) + return type(dataset)(_hf_dict=filtered) + + @staticmethod + def _build_grader(grader: Mapping | Model | None) -> Model: + """Resolve the ``grader`` task arg into a Model. + + Accepts a pre-built Model (tests / advanced configs) or a model-config + mapping (the YAML path, e.g. ``{model: gpt-5.2, api_base: ...}``). + Grading is mandatory — there is no deterministic fallback — so ``None`` + raises. + """ + if isinstance(grader, Model): + return grader + if isinstance(grader, Mapping): + return ChatModel(**grader) + raise ValueError( + "HLE requires an LLM judge. Pass `grader:` in the task args — a " + "model-config dict such as " + "{model: gpt-5.2, api_base: ..., api_key: ..., reasoning_effort: medium}." + ) + + @override + async def preprocess(self, raw, ctx): + content: list[dict] = [{"type": "text", "text": raw["question"]}] + if raw["image"]: # "" when not multi-modal + content.append({"type": "image_url", "image_url": {"url": raw["image"]}}) + return [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": content}, + ] + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre, n=self._n) + + @override + async def postprocess(self, inf, ctx): + return list(inf.texts) + + @override + async def feedback(self, post, ctx): + raw = ctx.raw_sample + question = raw["question"] + gold = raw["answer"] + grader_model = self._grader.meta()["model"] + + feedbacks: list[JudgeFeedback] = [] + for predicted in post: + prompt = JUDGE_PROMPT.format( + question=question, + correct_answer=gold, + response=predicted, + ) + out = await self._grader.agenerate(prompt) + reply = out.texts[0] if out.texts else "" + correct, confidence = parse_judge(reply) + feedbacks.append( + { + "correct": correct, + "confidence": confidence, + "gold": gold, + "predicted": predicted, + "grader_model": grader_model, + } + ) + return True, feedbacks + + @override + async def report(self, finals, fails): + correct = [fb["correct"] for f in finals for fb in (f.feedback_result or [])] + confidence = [ + fb["confidence"] for f in finals for fb in (f.feedback_result or []) + ] + # Denominator spans the full requested set; pipeline failures (candidate + # produced no gradeable answer) count as incorrect — matching upstream + # (n = total questions) and the *_gen family, not just graded attempts. + n = (len(finals) + len(fails)) * self._n + # Length-capped attempts: a reasoning model can burn the whole token + # budget and emit no answer, then get graded incorrect. Surface the + # count so the headline is self-documenting on this collapse-prone + # benchmark (upstream reports only accuracy). + truncated = 0 + for f in finals: + out = f.infer_result + if out is None or out.finish_reasons is None: + continue + truncated += sum(reason == "length" for reason in out.finish_reasons) + m = aggregate_metrics(correct, confidence, n) + return { + "score": m["accuracy"], + "accuracy": m["accuracy"], + "confidence_interval": m["confidence_interval"], + "calibration_error": m["calibration_error"], + "n": n, + "n_graded": len(correct), + "fails": len(fails), + "truncated": truncated, + } diff --git a/tests/unit/datasets/test_hle.py b/tests/unit/datasets/test_hle.py new file mode 100644 index 0000000..3e2aa6f --- /dev/null +++ b/tests/unit/datasets/test_hle.py @@ -0,0 +1,84 @@ +"""Unit tests for the HLE dataset wrapper. + +AI-Generated Code - Claude Opus 4.8 (Anthropic) +""" + +from unittest.mock import patch + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict +from datasets import Features, Image, Value + +import sieval.datasets.hle as hle_module +from sieval.core.datasets.meta import get_dataset_meta +from sieval.datasets.hle import HLE_REVISION, HLEDataset + + +def _row(image: str = "") -> dict: + return { + "id": "q1", + "question": "What is 2 + 2?", + "image": image, + "answer": "4", + "answer_type": "exactMatch", + "author_name": "author", + "rationale": "", + "raw_subject": "Math", + "category": "Math", + } + + +def _hf_dict(rows: int = 1) -> HFDatasetDict: + return HFDatasetDict({"test": HFDataset.from_list([_row()] * rows)}) + + +def test_source_pins_hf_revision(): + meta_source = get_dataset_meta(HLEDataset).source + assert meta_source == (f"hf:cais/hle@{HLE_REVISION}",) + + +def test_load_forwards_path_and_preserves_image_column(): + hf_dict = _hf_dict() + dataset = HLEDataset(_hf_dict=hf_dict) + with patch.object(hle_module, "load_dataset", return_value=hf_dict) as mock_load: + loaded = dataset.load("cais/hle") + + # Minimal loader: pass name_or_path straight through, keep the "test" split. + assert mock_load.call_args.args[0] == "cais/hle" + assert "test" in loaded + # The multimodal `image` column must be preserved (not welded to text-only). + assert "image" in loaded["test"].column_names + + +def test_load_disables_auxiliary_image_decoding(): + # image_preview / rationale_image are HF Image features (decode=True upstream); + # load() must disable decoding so a row fetch never requires Pillow. + features = Features( + { + "id": Value("string"), + "question": Value("string"), + "image": Value("string"), + "image_preview": Image(decode=True), + "answer": Value("string"), + "answer_type": Value("string"), + "author_name": Value("string"), + "rationale": Value("string"), + "rationale_image": Image(decode=True), + "raw_subject": Value("string"), + "category": Value("string"), + } + ) + row = {**_row(), "image_preview": None, "rationale_image": None} + hf = HFDatasetDict( + { + "test": HFDataset.from_dict( + {k: [row[k]] for k in features}, features=features + ) + } + ) + dataset = HLEDataset(_hf_dict=hf) + with patch.object(hle_module, "load_dataset", return_value=hf): + loaded = dataset.load("cais/hle") + + assert loaded["test"].features["image_preview"].decode is False + assert loaded["test"].features["rationale_image"].decode is False diff --git a/tests/unit/tasks/test_hle_0shot_gen.py b/tests/unit/tasks/test_hle_0shot_gen.py new file mode 100644 index 0000000..2133cad --- /dev/null +++ b/tests/unit/tasks/test_hle_0shot_gen.py @@ -0,0 +1,352 @@ +"""Unit tests for the HLE 0-shot generative task. + +AI-Generated Code - Claude Opus 4.8 (Anthropic) +""" + +import numpy as np +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.hle import ( + JUDGE_PROMPT, + SYSTEM_PROMPT, + aggregate_metrics, + calib_err, + parse_judge, +) +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import TaskContext +from sieval.datasets.hle import HLEDataset +from sieval.tasks.hle_0shot_gen import HLEZeroShotGenTask, JudgeFeedback + + +class _ScriptedChatModel(ChatModel): + """ChatModel returning a fixed reply, recording the last agenerate kwargs.""" + + def __init__(self, reply: str, model: str = "mock"): + super().__init__(model=model, api_key="fake") + self._reply = reply + self.last_kwargs: dict[str, object] = {} + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + _ = prompt + self.last_kwargs = dict(kwargs) + return ModelOutput(model=self.meta(), texts=[self._reply]) + + async def _alogprobs_impl( + self, prompt, *, max_tokens=1, logprobs=5, echo=True, temperature=0.0, **kwargs + ) -> ModelOutput: + _ = (prompt, max_tokens, logprobs, echo, temperature, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _row(image: str = "") -> dict: + return { + "id": "q1", + "question": "What is 2 + 2?", + "image": image, + "answer": "4", + "answer_type": "exactMatch", + "author_name": "author", + "rationale": "", + "raw_subject": "Math", + "category": "Math", + } + + +def _dataset(rows: list[dict]) -> HLEDataset: + return HLEDataset(_hf_dict=HFDatasetDict({"test": HFDataset.from_list(rows)})) + + +def _task( + grader_reply: str = "correct: yes\nconfidence: 90", + *, + rows: list[dict] | None = None, + text_only: bool = True, + n: int = 1, +): + dataset = _dataset(rows if rows is not None else [_row()]) + model = _ScriptedChatModel(reply="Answer: 4", model="candidate") + grader = _ScriptedChatModel(reply=grader_reply, model="judge-5.2") + task = HLEZeroShotGenTask(dataset, model, grader=grader, n=n, text_only=text_only) + return task, model, grader + + +# --- grader is mandatory; no deterministic fallback --- + + +def test_build_grader_requires_config(): + with pytest.raises(ValueError, match="requires an LLM judge"): + HLEZeroShotGenTask._build_grader(None) + + +def test_build_grader_accepts_mapping_and_model(): + built = HLEZeroShotGenTask._build_grader({"model": "gpt-5.2", "api_key": "fake"}) + assert isinstance(built, ChatModel) + existing = _ScriptedChatModel(reply="x") + assert HLEZeroShotGenTask._build_grader(existing) is existing + + +# --- text-only selection drops image questions --- + + +def test_text_only_keeps_only_text_questions(): + task, _, _ = _task(rows=[_row(), _row(image="data:image/png;base64,AAAA")]) + assert task.dataset.test_set is not None + assert len(task.dataset.test_set) == 1 + assert task.dataset.test_set[0]["image"] == "" + + +def test_full_set_keeps_image_questions(): + task, _, _ = _task( + rows=[_row(), _row(image="data:image/png;base64,AAAA")], text_only=False + ) + assert task.dataset.test_set is not None + assert len(task.dataset.test_set) == 2 + + +def test_text_only_all_multimodal_raises(): + with pytest.raises(ValueError, match="empty 'test' split"): + _task(rows=[_row(image="data:image/png;base64,AAAA")]) + + +# --- preprocess: HLE system prompt + user content blocks (mirrors format_message) --- + + +@pytest.mark.anyio +async def test_preprocess_text_only_message(): + task, _, _ = _task() + messages = await task.preprocess(_row(), TaskContext(sample_id=0)) + assert messages == [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": [{"type": "text", "text": "What is 2 + 2?"}]}, + ] + + +@pytest.mark.anyio +async def test_preprocess_attaches_image_block(): + task, _, _ = _task(text_only=False) + raw = _row(image="data:image/png;base64,AAAA") + messages = await task.preprocess(raw, TaskContext(sample_id=0)) + user_content = messages[1]["content"] + assert user_content[0] == {"type": "text", "text": "What is 2 + 2?"} + assert user_content[1] == { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}, + } + + +# --- infer forwards ONLY n (no decode-param injection) --- + + +@pytest.mark.anyio +async def test_infer_forwards_only_n(): + task, model, _ = _task(n=1) + await task.infer([{"role": "user", "content": "q"}], TaskContext(sample_id=0)) + # Decode params (temperature/top_p/max_tokens) must come from the model + # layer, never from the task — infer passes n and nothing else. + assert model.last_kwargs == {"n": 1} + + +# --- feedback: parse judge correct/confidence, record provenance --- + + +@pytest.mark.anyio +async def test_feedback_parses_correct_and_confidence(): + task, _, _ = _task(grader_reply="correct: yes\nconfidence: 90") + ctx = TaskContext(sample_id=0, raw_sample=_row()) + finalize, feedbacks = await task.feedback(["Answer: 4"], ctx) + + assert finalize is True + fb: JudgeFeedback = feedbacks[0] + assert fb["correct"] is True + assert fb["confidence"] == 90 + assert fb["gold"] == "4" + assert fb["predicted"] == "Answer: 4" + assert fb["grader_model"] == "judge-5.2" + + +@pytest.mark.anyio +async def test_feedback_unparseable_reply_is_incorrect_default_confidence(): + task, _, _ = _task(grader_reply="the judge rambled without the fields") + ctx = TaskContext(sample_id=0, raw_sample=_row()) + _, feedbacks = await task.feedback(["whatever"], ctx) + assert feedbacks[0]["correct"] is False + assert feedbacks[0]["confidence"] == 100 + + +# --- report: accuracy over the full requested set (fails in denominator) --- + + +def _finals(grades: list[tuple[bool, int]]) -> list[TaskContext]: + return [ + TaskContext( + sample_id=i, + feedback_result=[ + { + "correct": c, + "confidence": conf, + "gold": "", + "predicted": "", + "grader_model": "m", + } + ], + ) + for i, (c, conf) in enumerate(grades) + ] + + +@pytest.mark.anyio +async def test_report_accuracy_and_counts_fails_in_denominator(): + task, _, _ = _task() # n=1 + finals = _finals([(True, 90), (False, 40)]) + fails = [TaskContext(sample_id=10)] + report = await task.report(finals, fails) + + # n = (2 finals + 1 fail) * 1 = 3; 1 correct => 33.33%. + # The old len(finals)=2 denominator would give 50.0, so this discriminates. + assert report["n"] == 3 + assert report["n_graded"] == 2 + assert report["fails"] == 1 + # No infer_result on these contexts -> no truncation surfaced. + assert report["truncated"] == 0 + assert report["accuracy"] == pytest.approx(33.33, abs=1e-2) + assert report["score"] == report["accuracy"] + + +@pytest.mark.anyio +async def test_report_fails_weighted_by_n(): + task, _, _ = _task(n=2) + # One finalized sample carrying its n=2 judged attempts (both correct). + fb = { + "correct": True, + "confidence": 90, + "gold": "", + "predicted": "", + "grader_model": "m", + } + finals = [TaskContext(sample_id=0, feedback_result=[fb, dict(fb)])] + fails = [TaskContext(sample_id=5)] + report = await task.report(finals, fails) + # n = (1 final + 1 fail) * 2 = 4; 2 correct => 50.0%. + # An unweighted (n=1) denominator would give 3 and 66.67%. + assert report["n"] == 4 + assert report["n_graded"] == 2 + assert report["accuracy"] == pytest.approx(50.0) + + +@pytest.mark.anyio +async def test_report_counts_truncated_outputs(): + # A length-capped attempt (finish_reason "length") is surfaced as + # `truncated` so the accuracy headline is self-documenting on this + # collapse-prone benchmark. Counted per-attempt from infer_result. + task, model, _ = _task() # n=1 + meta = model.meta() + fb = { + "correct": False, + "confidence": 100, + "gold": "", + "predicted": "", + "grader_model": "m", + } + finals = [ + TaskContext( + sample_id=0, + infer_result=ModelOutput(model=meta, texts=[""], finish_reasons=["length"]), + feedback_result=[dict(fb)], + ), + TaskContext( + sample_id=1, + infer_result=ModelOutput( + model=meta, texts=["Answer: 4"], finish_reasons=["stop"] + ), + feedback_result=[{**fb, "correct": True}], + ), + ] + report = await task.report(finals, []) + assert report["truncated"] == 1 + assert report["n_graded"] == 2 + + +@pytest.mark.anyio +async def test_report_empty_is_zero(): + task, _, _ = _task() + report = await task.report([], []) + assert report["n"] == 0 + assert report["accuracy"] == 0.0 + assert report["calibration_error"] == 0.0 + assert report["truncated"] == 0 + + +# --- prompt fidelity: byte-for-byte pins on the vendored HLE prompts --- +# These lock the reproduction invariant so any drift from upstream +# (centerforaisafety/hle @ 26dca2e) fails loudly. `test_preprocess_*` above +# compare against the constants by reference and cannot catch such drift. + + +def test_system_prompt_pinned(): + assert SYSTEM_PROMPT == ( + "Your response should be in the following format:\n" + "Explanation: {your explanation for your answer choice}\n" + "Answer: {your chosen answer}\n" + "Confidence: {your confidence score between 0% and 100% for your answer}" + ) + + +def test_judge_prompt_pinned(): + # Upstream ships a duplicated-word typo and pipe-escaped percent signs; + # both are preserved verbatim. + assert "i.e. if there if there is any inconsistency" in JUDGE_PROMPT + assert r"confidence score between 0|\%| and 100|\%| from [response]" in JUDGE_PROMPT + assert "extracted_final_answer:" in JUDGE_PROMPT + for field in ("{question}", "{response}", "{correct_answer}"): + assert field in JUDGE_PROMPT + + +# --- metric kernel: parse_judge, calib_err, aggregate_metrics --- + + +def test_parse_judge_last_field_wins(): + assert parse_judge("correct: yes\nconfidence: 85") == (True, 85) + assert parse_judge("correct: no") == (False, 100) + assert parse_judge("no recognizable fields") == (False, 100) + # reasoning may mention "correct:"; the trailing field value wins. + assert parse_judge("reasoning: correct: no\ncorrect: yes\nconfidence: 30") == ( + True, + 30, + ) + # `\b` anchor: "incorrect: yes" must NOT be read as the `correct` field. + # Without the anchor the substring "correct: yes" would match -> (True, 100); + # with no real verdict field the parse must default to (False, 100). + assert parse_judge("extracted_final_answer: 42 is incorrect: yes") == (False, 100) + + +def test_calib_err_matches_hand_computation(): + # beta=2 forces two bins over four samples so the first bin is scored + # (upstream excludes the final bin via range(len(bins) - 1)). + confidence = np.array([0.1, 0.2, 0.9, 0.95]) + correct = np.array([0, 0, 1, 1]) + # bin[0] conf mean 0.15, correct mean 0 -> diff 0.15; + # cerr = sqrt(2/4 * 0.15**2) = 0.106066... + assert calib_err(confidence, correct, p="2", beta=2) == pytest.approx( + 0.106066, abs=1e-5 + ) + + +def test_aggregate_metrics_accuracy_ci_and_calibration_guard(): + # 1 correct of n=4 -> 25.0%; Wald half-width = 1.96*sqrt(25*75/4) = 42.44. + m = aggregate_metrics([True, False], [100, 50], n=4) + assert m["accuracy"] == pytest.approx(25.0) + assert m["confidence_interval"] == pytest.approx(42.44, abs=1e-2) + # Fewer than BETA judged records -> calibration guarded to 0.0. + assert m["calibration_error"] == 0.0 + + +def test_aggregate_metrics_zero_n(): + assert aggregate_metrics([], [], n=0) == { + "accuracy": 0.0, + "confidence_interval": 0.0, + "calibration_error": 0.0, + }