Add IFBench eval suite (ifeval_ood + ifeval_mt) aligned with oe-eval-internal

pyproject.toml

@@ -31,6 +31,9 @@ dependencies = [
     "textual-plot>=0.10.1",
     "tree-sitter>=0.24.0",
     "tree-sitter-python>=0.23.0,<0.26.0",
+    "ifbench @ git+https://github.com/allenai/IFBench.git@finbarr/clean-install",
+    # syllapy (transitive via ifbench) imports pkg_resources, which setuptools>=81 removes.
+    "setuptools<81",
 ]
 
 [project.urls]
@@ -222,6 +225,8 @@ allowed-unresolved-imports = [
     "google.cloud.**",
     "matplotlib.**",
     "scipy.**",
+    "instructions_registry.**",
+    "instructions.**",
 ]
 
 [tool.gantry]

src/olmo_eval/common/metrics/__init__.py

@@ -19,6 +19,12 @@
     SQuADF1Metric,
     ToolAccuracyMetric,
 )
+from .ifeval import (
+    IFEvalInstLooseAccuracy,
+    IFEvalInstStrictAccuracy,
+    IFEvalPromptLooseAccuracy,
+    IFEvalPromptStrictAccuracy,
+)
 
 __all__ = [
     "AccuracyMetric",
@@ -27,6 +33,10 @@
     "CorpusPerplexityMetric",
     "F1Metric",
     "GreedyAccuracyMetric",
+    "IFEvalInstLooseAccuracy",
+    "IFEvalInstStrictAccuracy",
+    "IFEvalPromptLooseAccuracy",
+    "IFEvalPromptStrictAccuracy",
     "LogprobMCAccuracyMetric",
     "LogprobPerCharMCAccuracyMetric",
     "LogprobPerTokenMCAccuracyMetric",

src/olmo_eval/common/metrics/ifeval.py

@@ -0,0 +1,87 @@
+"""IFBench / IFEval metrics.
+
+All four metrics share :class:`IFEvalScorer`, which writes per-instruction
+strict and loose pass lists to ``output.metadata["ifeval"]``. Each metric
+aggregates that side-band data along the prompt or instruction axis.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import ClassVar
+
+from olmo_eval.common.metrics.base import Metric
+from olmo_eval.common.scorers import IFEvalScorer, Scorer
+from olmo_eval.common.types import Response
+
+
+def _iter_results(responses: Sequence[Response], key: str) -> list[list[bool]]:
+    out: list[list[bool]] = []
+    for response in responses:
+        if not response.outputs:
+            out.append([])
+            continue
+        meta = response.outputs[0].metadata or {}
+        ifeval = meta.get("ifeval") or {}
+        out.append(list(ifeval.get(key, [])))
+    return out
+
+
+def _prompt_level(results: list[list[bool]]) -> float:
+    if not results:
+        return 0.0
+    correct = sum(1 for r in results if r and all(r))
+    return correct / len(results)
+
+
+def _instruction_level(results: list[list[bool]]) -> float:
+    total = sum(len(r) for r in results)
+    if total == 0:
+        return 0.0
+    correct = sum(1 for r in results for v in r if v)
+    return correct / total
+
+
+@dataclass(frozen=True, slots=True)
+class IFEvalPromptStrictAccuracy(Metric):
+    """Fraction of prompts where every instruction passes under strict scoring."""
+
+    name: ClassVar[str] = "prompt_level_strict_acc"
+    scorer: ClassVar[type[Scorer]] = IFEvalScorer
+
+    def compute(self, responses: Sequence[Response]) -> float:
+        return _prompt_level(_iter_results(responses, "strict"))
+
+
+@dataclass(frozen=True, slots=True)
+class IFEvalPromptLooseAccuracy(Metric):
+    """Fraction of prompts where every instruction passes under loose scoring."""
+
+    name: ClassVar[str] = "prompt_level_loose_acc"
+    scorer: ClassVar[type[Scorer]] = IFEvalScorer
+
+    def compute(self, responses: Sequence[Response]) -> float:
+        return _prompt_level(_iter_results(responses, "loose"))
+
+
+@dataclass(frozen=True, slots=True)
+class IFEvalInstStrictAccuracy(Metric):
+    """Fraction of individual instructions passing under strict scoring."""
+
+    name: ClassVar[str] = "inst_level_strict_acc"
+    scorer: ClassVar[type[Scorer]] = IFEvalScorer
+
+    def compute(self, responses: Sequence[Response]) -> float:
+        return _instruction_level(_iter_results(responses, "strict"))
+
+
+@dataclass(frozen=True, slots=True)
+class IFEvalInstLooseAccuracy(Metric):
+    """Fraction of individual instructions passing under loose scoring."""
+
+    name: ClassVar[str] = "inst_level_loose_acc"
+    scorer: ClassVar[type[Scorer]] = IFEvalScorer
+
+    def compute(self, responses: Sequence[Response]) -> float:
+        return _instruction_level(_iter_results(responses, "loose"))

src/olmo_eval/common/scorers/__init__.py

@@ -15,6 +15,7 @@
 )
 from .code_execution import CodeExecutionScorer, MultiplEScorer
 from .execution import ContextScorer, ExecutionScorer, SandboxRequiredError
+from .ifeval import IFEvalScorer
 from .llm_judge import (
     JudgeFn,
     LLMJudgeScorer,
@@ -45,6 +46,7 @@
     "ExactMatchScorer",
     "ExecutionScorer",
     "F1Scorer",
+    "IFEvalScorer",
     "JudgeFn",
     "LLMJudgeScorer",
     "LogprobScorer",

src/olmo_eval/common/scorers/ifeval.py

@@ -0,0 +1,104 @@
+"""Scorer for IFBench / IFEval instruction-following evaluation.
+
+Uses the ``ifbench`` package registry, which covers the original IFEval
+(DEFAULT) verifiers, the OOD verifiers used by ``allenai/IFBench_test2``,
+and the verifiers used by the multi-turn ``VGraf/ifeval_mt`` slices. The
+scorer evaluates a response against per-instance instructions (looked up
+in ``instance.metadata["instruction_id_list"]`` / ``"kwargs"``) and writes
+both strict and loose pass/fail lists for each instruction into
+``output.metadata["ifeval"]``. The four IFEval metrics consume that field.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from olmo_eval.common.scorers.base import Scorer
+from olmo_eval.common.types import Instance, LMOutput
+
+
+def _loose_response_variants(response: str) -> list[str]:
+    """Generate the eight response variants used by upstream loose scoring."""
+    lines = response.split("\n")
+    remove_first = "\n".join(lines[1:]).strip()
+    remove_last = "\n".join(lines[:-1]).strip()
+    remove_both = "\n".join(lines[1:-1]).strip()
+    return [
+        response,
+        response.replace("*", ""),
+        remove_first,
+        remove_last,
+        remove_both,
+        remove_first.replace("*", ""),
+        remove_last.replace("*", ""),
+        remove_both.replace("*", ""),
+    ]
+
+
+def _check_one(
+    instruction_cls: Any,
+    instruction_id: str,
+    kwargs: dict[str, Any],
+    prompt: str,
+    response: str,
+) -> bool:
+    instruction = instruction_cls(instruction_id)
+    cleaned_kwargs = {k: v for k, v in kwargs.items() if v is not None}
+    arg_keys = instruction.get_instruction_args_keys()
+    if "prompt_to_repeat" in arg_keys and not cleaned_kwargs.get("prompt_to_repeat"):
+        cleaned_kwargs["prompt_to_repeat"] = prompt
+    instruction.build_description(**cleaned_kwargs)
+    args = instruction.get_instruction_args()
+    if args and "prompt" in args:
+        instruction.build_description(prompt=prompt)
+    return bool(response.strip()) and bool(instruction.check_following(response))
+
+
+@dataclass(frozen=True)
+class IFEvalScorer(Scorer):
+    """Run IFBench/IFEval instruction verifiers against a response.
+
+    The numeric ``score()`` return is the prompt-level loose accuracy (1.0 if
+    every instruction passes under at least one loose variant, else 0.0). The
+    full strict + loose pass lists are written to ``output.metadata["ifeval"]``
+    so the four metric classes can derive prompt/inst × strict/loose figures.
+    """
+
+    name: ClassVar[str] = "ifeval"
+
+    def score(self, instance: Instance, output: LMOutput) -> float:
+        instruction_ids: list[str] = instance.metadata.get("instruction_id_list", [])
+        kwargs_list: list[dict[str, Any]] = instance.metadata.get("kwargs", [])
+        prompt: str = instance.metadata.get("prompt", instance.question)
+        response: str = output.text or ""
+
+        strict_results: list[bool] = []
+        loose_results: list[bool] = []
+
+        if instruction_ids:
+            from ifbench import instructions_registry
+
+            registry = instructions_registry.INSTRUCTION_DICT
+            loose_variants = _loose_response_variants(response)
+            for inst_id, inst_kwargs in zip(instruction_ids, kwargs_list, strict=True):
+                instruction_cls = registry[inst_id]
+                strict_results.append(
+                    _check_one(instruction_cls, inst_id, inst_kwargs, prompt, response)
+                )
+                loose_pass = any(
+                    _check_one(instruction_cls, inst_id, inst_kwargs, prompt, variant)
+                    for variant in loose_variants
+                )
+                loose_results.append(loose_pass)
+
+        if output.metadata is None:
+            output.metadata = {}
+        output.metadata["ifeval"] = {
+            "strict": strict_results,
+            "loose": loose_results,
+        }
+
+        if not loose_results:
+            return 0.0
+        return 1.0 if all(loose_results) else 0.0

src/olmo_eval/evals/suites/ifbench.py

@@ -0,0 +1,13 @@
+"""IFBench (Tulu) instruction-following suite."""
+
+from olmo_eval.evals.suites.registry import make_suite
+
+IFBENCH = make_suite(
+    "ifbench",
+    (
+        "ifeval_mt_wildchat_unused_withRewrite",
+        "ifeval_mt_ood_wildchat_unused_withRewrite",
+        "ifeval_ood",
+    ),
+    description="IFBench (Tulu): OOD + multi-turn instruction following",
+)

src/olmo_eval/evals/tasks/ifeval_mt.py

@@ -0,0 +1,104 @@
+"""IFEval-MT: multi-turn instruction following from ``VGraf/ifeval_mt``.
+
+Each row of the dataset is a multi-turn conversation that ends in a user turn
+asking the assistant to rewrite/repeat its prior reply under a list of
+instructions. ``instruction_id_list`` and ``kwargs`` are scored against the
+final assistant reply using the same IFEval verifiers as ``ifeval_ood``.
+
+Two registered variants follow ``IFBENCH_MT_TASKS`` in oe-eval-internal:
+
+- ``ifeval_mt_wildchat_unused_withRewrite`` (HF subset
+  ``wildchat_unused_withRewrite``)
+- ``ifeval_mt_ood_wildchat_unused_withRewrite`` (HF subset
+  ``ood_wildchat_unused_withRewrite``)
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+from olmo_eval.common.metrics import (
+    IFEvalInstLooseAccuracy,
+    IFEvalInstStrictAccuracy,
+    IFEvalPromptLooseAccuracy,
+    IFEvalPromptStrictAccuracy,
+)
+from olmo_eval.common.types import (
+    Instance,
+    LMOutput,
+    LMRequest,
+    RequestType,
+    SamplingParams,
+    Split,
+)
+from olmo_eval.data import DataSource
+from olmo_eval.evals.tasks.common import Task, register
+
+_PRIMARY_METRIC = IFEvalPromptLooseAccuracy()
+_DATASET_PATH = "VGraf/ifeval_mt"
+_SAMPLING_PARAMS = SamplingParams(
+    max_tokens=2048,
+    temperature=0.0,
+    do_sample=False,
+)
+
+
+class _IFEvalMTBase(Task):
+    split = Split.TEST
+    metrics = (
+        IFEvalPromptStrictAccuracy(),
+        IFEvalPromptLooseAccuracy(),
+        IFEvalInstStrictAccuracy(),
+        IFEvalInstLooseAccuracy(),
+    )
+    primary_metric = _PRIMARY_METRIC
+    sampling_params = _SAMPLING_PARAMS
+
+    @property
+    def instances(self) -> Iterator[Instance]:
+        yield from self._load_instances_cached()
+
+    @property
+    def request_type(self) -> RequestType:
+        return RequestType.CHAT
+
+    def process_doc(self, doc: dict[str, Any], index: int = 0) -> Instance | None:
+        prompt = doc["prompt"]
+        instruction_id_list = list(doc.get("instruction_id_list") or [])
+        raw_kwargs = doc.get("kwargs") or []
+        kwargs_list = [{k: v for k, v in (kw or {}).items() if v is not None} for kw in raw_kwargs]
+        messages = tuple({"role": m["role"], "content": m["content"]} for m in doc["messages"])
+        return Instance(
+            question=prompt,
+            gold_answer=None,
+            metadata={
+                "id": doc.get("id", doc.get("key", index)),
+                "key": doc.get("key", doc.get("id", index)),
+                "prompt": prompt,
+                "instruction_id_list": instruction_id_list,
+                "kwargs": kwargs_list,
+                "messages": messages,
+            },
+        )
+
+    def format_request(self, instance: Instance) -> LMRequest:
+        return LMRequest(
+            request_type=RequestType.CHAT,
+            messages=tuple(instance.metadata["messages"]),
+        )
+
+    def extract_answer(self, output: LMOutput) -> str:
+        return output.text
+
+
+@register("ifeval_mt_wildchat_unused_withRewrite")
+class IFEvalMTWildchatUnusedWithRewrite(_IFEvalMTBase):
+    data_source = DataSource(path=_DATASET_PATH, subset="wildchat_unused_withRewrite", split="test")
+
+
+@register("ifeval_mt_ood_wildchat_unused_withRewrite")
+class IFEvalMTOODWildchatUnusedWithRewrite(_IFEvalMTBase):
+    data_source = DataSource(
+        path=_DATASET_PATH, subset="ood_wildchat_unused_withRewrite", split="test"
+    )