From 032c61b993372fbb7b4503e38f229139109afecf Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Thu, 23 Jul 2026 15:11:13 +0800 Subject: [PATCH 1/2] feat(scicode): add SciCode dataset + 0-shot generation task --- .gitignore | 7 + pdm.lock | 53 +++- pyproject.toml | 1 + sieval/community/scicode/__init__.py | 49 ++++ sieval/community/scicode/_cmp_upstream.py | 89 ++++++ sieval/community/scicode/harness.py | 83 ++++++ sieval/community/scicode/parse.py | 171 +++++++++++ sieval/community/scicode/prompts.py | 179 ++++++++++++ sieval/datasets/__init__.pyi | 6 + sieval/datasets/scicode.py | 98 +++++++ sieval/meta/index.json | 49 ++++ sieval/tasks/__init__.pyi | 4 + sieval/tasks/scicode_0shot_gen.py | 323 +++++++++++++++++++++ tests/unit/datasets/test_scicode.py | 87 ++++++ tests/unit/tasks/test_scicode_0shot_gen.py | 310 ++++++++++++++++++++ 15 files changed, 1505 insertions(+), 4 deletions(-) create mode 100644 sieval/community/scicode/__init__.py create mode 100644 sieval/community/scicode/_cmp_upstream.py create mode 100644 sieval/community/scicode/harness.py create mode 100644 sieval/community/scicode/parse.py create mode 100644 sieval/community/scicode/prompts.py create mode 100644 sieval/datasets/scicode.py create mode 100644 sieval/tasks/scicode_0shot_gen.py create mode 100644 tests/unit/datasets/test_scicode.py create mode 100644 tests/unit/tasks/test_scicode_0shot_gen.py diff --git a/.gitignore b/.gitignore index 40963d77..6bce3a15 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,10 @@ docs/handover/ # Local-only model reference (not for OSS distribution) models.yaml + +# Local-only eval run configs (may contain live API keys) +scicode-kimi.yaml +scicode-kimi-slice.yaml +scicode-kimi-p13.yaml +scicode-kimi-bg-slice.yaml +scicode-kimi-full-bg.yaml diff --git a/pdm.lock b/pdm.lock index 4002a168..5b1f6f3d 100644 --- a/pdm.lock +++ b/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "dev", "drop", "ifbench", "ifeval", "math", "t-eval", "test"] +groups = ["default", "dev", "drop", "ifbench", "ifeval", "math", "scicode", "t-eval", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:aadbe110ed98781ee5c51ae38bd40780304ed26da3c371a0fd3f5ff1fd1256ef" +content_hash = "sha256:36f42031d4d603c12b7ded5f2c50514b69bc81e3a07bc0f4026f16dda2390677" [[metadata.targets]] requires_python = ">=3.12,<3.15" @@ -694,6 +694,51 @@ files = [ {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, ] +[[package]] +name = "h5py" +version = "3.16.0" +requires_python = ">=3.10" +summary = "Read and write HDF5 files from Python" +groups = ["scicode"] +dependencies = [ + "numpy>=1.21.2", +] +files = [ + {file = "h5py-3.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c5313566f4643121a78503a473f0fb1e6dcc541d5115c44f05e037609c565c4d"}, + {file = "h5py-3.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42b012933a83e1a558c673176676a10ce2fd3759976a0fedee1e672d1e04fc9d"}, + {file = "h5py-3.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ff24039e2573297787c3063df64b60aab0591980ac898329a08b0320e0cf2527"}, + {file = "h5py-3.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:dfc21898ff025f1e8e67e194965a95a8d4754f452f83454538f98f8a3fcb207e"}, + {file = "h5py-3.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:698dd69291272642ffda44a0ecd6cd3bda5faf9621452d255f57ce91487b9794"}, + {file = "h5py-3.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2b2c02b0a160faed5fb33f1ba8a264a37ee240b22e049ecc827345d0d9043074"}, + {file = "h5py-3.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:96b422019a1c8975c2d5dadcf61d4ba6f01c31f92bbde6e4649607885fe502d6"}, + {file = "h5py-3.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:39c2838fb1e8d97bcf1755e60ad1f3dd76a7b2a475928dc321672752678b96db"}, + {file = "h5py-3.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:370a845f432c2c9619db8eed334d1e610c6015796122b0e57aa46312c22617d9"}, + {file = "h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42108e93326c50c2810025aade9eac9d6827524cdccc7d4b75a546e5ab308edb"}, + {file = "h5py-3.16.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:099f2525c9dcf28de366970a5fb34879aab20491589fa89ce2863a84218bb524"}, + {file = "h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9300ad32dea9dfc5171f94d5f6948e159ed93e4701280b0f508773b3f582f402"}, + {file = "h5py-3.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:171038f23bccddfc23f344cadabdfc9917ff554db6a0d417180d2747fe4c75a7"}, + {file = "h5py-3.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7e420b539fb6023a259a1b14d4c9f6df8cf50d7268f48e161169987a57b737ff"}, + {file = "h5py-3.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:18f2bbcd545e6991412253b98727374c356d67caa920e68dc79eab36bf5fedad"}, + {file = "h5py-3.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:656f00e4d903199a1d58df06b711cf3ca632b874b4207b7dbec86185b5c8c7d4"}, + {file = "h5py-3.16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9c9d307c0ef862d1cd5714f72ecfafe0a5d7529c44845afa8de9f46e5ba8bd65"}, + {file = "h5py-3.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c1eff849cdd53cbc73c214c30ebdb6f1bb8b64790b4b4fc36acdb5e43570210"}, + {file = "h5py-3.16.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2c04d129f180019e216ee5f9c40b78a418634091c8782e1f723a6ca3658b965"}, + {file = "h5py-3.16.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4360f15875a532bc7b98196c7592ed4fc92672a57c0a621355961cafb17a6dd"}, + {file = "h5py-3.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3fae9197390c325e62e0a1aa977f2f62d994aa87aab182abbea85479b791197c"}, + {file = "h5py-3.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43259303989ac8adacc9986695b31e35dba6fd1e297ff9c6a04b7da5542139cc"}, + {file = "h5py-3.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:fa48993a0b799737ba7fd21e2350fa0a60701e58180fae9f2de834bc39a147ab"}, + {file = "h5py-3.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:1897a771a7f40d05c262fc8f37376ec37873218544b70216872876c627640f63"}, + {file = "h5py-3.16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15922e485844f77c0b9d275396d435db3baa58292a9c2176a386e072e0cf2491"}, + {file = "h5py-3.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:df02dd29bd247f98674634dfe41f89fd7c16ba3d7de8695ec958f58404a4e618"}, + {file = "h5py-3.16.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0f456f556e4e2cebeebd9d66adf8dc321770a42593494a0b6f0af54a7567b242"}, + {file = "h5py-3.16.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3e6cb3387c756de6a9492d601553dffea3fe11b5f22b443aac708c69f3f55e16"}, + {file = "h5py-3.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8389e13a1fd745ad2856873e8187fd10268b2d9677877bb667b41aebd771d8b7"}, + {file = "h5py-3.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:346df559a0f7dcb31cf8e44805319e2ab24b8957c45e7708ce503b2ec79ba725"}, + {file = "h5py-3.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4c6ab014ab704b4feaa719ae783b86522ed0bf1f82184704ed3c9e4e3228796e"}, + {file = "h5py-3.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:faca8fb4e4319c09d83337adc80b2ca7d5c5a343c2d6f1b6388f32cfecca13c1"}, + {file = "h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738"}, +] + [[package]] name = "hf-xet" version = "1.2.0" @@ -1488,7 +1533,7 @@ name = "numpy" version = "2.2.0" requires_python = ">=3.10" summary = "Fundamental package for array computing in Python" -groups = ["default", "drop", "t-eval"] +groups = ["default", "drop", "scicode", "t-eval"] files = [ {file = "numpy-2.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cff210198bb4cae3f3c100444c5eaa573a823f05c253e7188e1362a5555235b3"}, {file = "numpy-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b92a5828bd4d9aa0952492b7de803135038de47343b2aa3cc23f3b71a3dc4e"}, @@ -2585,7 +2630,7 @@ name = "scipy" version = "1.16.3" requires_python = ">=3.11" summary = "Fundamental algorithms for scientific computing in Python" -groups = ["drop", "t-eval"] +groups = ["drop", "scicode", "t-eval"] dependencies = [ "numpy<2.6,>=1.25.2", ] diff --git a/pyproject.toml b/pyproject.toml index 4de8eb8d..37814686 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ math = [ "math-verify[antlr4-11-0]==0.8.0", "regex>=2024.0.0", ] +scicode = ["h5py>=3.11", "numpy<=2.2", "scipy>=1.16.3"] t-eval = ["numpy<=2.2", "sentence-transformers>=5.1.2"] [project.scripts] diff --git a/sieval/community/scicode/__init__.py b/sieval/community/scicode/__init__.py new file mode 100644 index 00000000..7cf35781 --- /dev/null +++ b/sieval/community/scicode/__init__.py @@ -0,0 +1,49 @@ +# Copyright 2024 The SciCode authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Vendored SciCode evaluation assets (github.com/scicode-bench/SciCode @ 69a8cfc). + +Local adaptation of the upstream prompt templates, code/HDF5 parsers, comparison +helpers, and the three scientist-authored gold steps, plus a sandbox-program +builder that inlines h5 targets so sieval's stateless code-eval service can run +the tests. +""" + +from sieval.community.scicode.harness import ( + build_test_program, + encode_targets, +) +from sieval.community.scicode.parse import ( + extract_function_name, + extract_python_script, + get_function_from_code, + process_hdf5_to_tuple, +) +from sieval.community.scicode.prompts import ( + generate_prompt_with_steps, + is_special_step, + special_step_code, +) + +__all__ = [ + "build_test_program", + "encode_targets", + "extract_function_name", + "extract_python_script", + "get_function_from_code", + "process_hdf5_to_tuple", + "generate_prompt_with_steps", + "is_special_step", + "special_step_code", +] diff --git a/sieval/community/scicode/_cmp_upstream.py b/sieval/community/scicode/_cmp_upstream.py new file mode 100644 index 00000000..6db2cebb --- /dev/null +++ b/sieval/community/scicode/_cmp_upstream.py @@ -0,0 +1,89 @@ +# Copyright 2024 The SciCode authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# adapted verbatim from https://github.com/scicode-bench/SciCode/blob/69a8cfc829fe8788a426ce8b5de6292366dce7ef/src/scicode/compare/cmp.py +import numpy as np +import scipy.sparse +import sympy + + +def are_dicts_close(dict1, dict2, atol=1e-8, rtol=1e-5): + dict1 = process_symbol_in_dict(dict1) + dict2 = process_symbol_in_dict(dict2) + # Check if both dictionaries have the same keys + if dict1.keys() != dict2.keys(): + return False + + # Check if the corresponding values are close + for key in dict1: + value1 = dict1[key] + value2 = dict2[key] + if isinstance(value1, (sympy.Symbol, str)): + if not value1 == value2: + return False + elif isinstance(value1, (scipy.sparse.csr_matrix, scipy.sparse.csc_matrix, + scipy.sparse.bsr_matrix, scipy.sparse.coo_matrix)): + value1 = value1.toarray() + value2 = value2.toarray() + if not np.allclose(value1, value2, atol=atol, rtol=rtol): + return False + # Use np.allclose to compare values + else: + try: + if not np.allclose(value1, value2, atol=atol, rtol=rtol): + return False + except ValueError: + if not value1 == value2: + return False + + return True + +def process_symbol_in_dict(dict): + new_dict = {} + for key, value in dict.items(): + new_dict[key] = value + if isinstance(value, sympy.Symbol): + new_dict[key] = str(value) + if isinstance(key, sympy.Symbol): + new_dict[str(key)] = dict[key] + new_dict.pop(key) + return new_dict + +def are_csc_matrix_close(matrix1, matrix2): + dense1 = matrix1.toarray() + dense2 = matrix2.toarray() + return np.allclose(dense1, dense2) + +def cmp_tuple_or_list(var1, var2): + if len(var1) != len(var2): + return False + for v1, v2 in zip(var1, var2): + if isinstance(v1, dict): + if not are_dicts_close(v1, v2): + return False + elif isinstance(v1, (scipy.sparse.csr_matrix, scipy.sparse.csc_matrix)): + if not are_csc_matrix_close(v1, v2): + return False + elif isinstance(v1, bool): + if not v1 == v2: + return False + else: + try: + if not np.allclose(v1, v2): + return False + except ValueError as e: + print(e) + if not v1 == v2: + return False + return True diff --git a/sieval/community/scicode/harness.py b/sieval/community/scicode/harness.py new file mode 100644 index 00000000..468577e6 --- /dev/null +++ b/sieval/community/scicode/harness.py @@ -0,0 +1,83 @@ +"""Assemble a single self-contained test program for one SciCode sub-step. + +Upstream (eval/scripts/test_generated_code.py) runs each step inside an +environment where ``scicode`` is installed and ``test_data.h5`` is on disk, then +appends ``targets = process_hdf5_to_tuple(step_id, n)`` followed by each test +case. sieval's code-eval sandbox is stateless, so instead we: + + * read the numeric targets from the h5 on the eval side and inline them as a + pickled+zlib+base64 blob (mirrors how LiveCodeBench inlines its private + tests), and + * register an in-process ``scicode.compare.cmp`` module so the test bodies' + ``from scicode.compare.cmp import cmp_tuple_or_list`` imports resolve. + +The concatenated function code (dependencies + prior steps + current step) is +supplied verbatim by the caller. The sandbox must provide the scientific stack +the problems import (numpy / scipy / sympy / ...); h5py is NOT needed there. + +AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic) +""" +import base64 +import pickle +import zlib +from importlib import resources + + +def _cmp_source() -> str: + return ( + resources.files("sieval.community.scicode") + .joinpath("_cmp_upstream.py") + .read_text(encoding="utf-8") + ) + + +# Registers the vendored comparison helpers under the module path the upstream +# test bodies import from, without requiring `scicode` to be installed. +_SHIM_FOOTER = """ +import sys as _sys, types as _types + +_scicode_cmp = _types.ModuleType("scicode.compare.cmp") +for _name in ( + "cmp_tuple_or_list", + "are_dicts_close", + "are_csc_matrix_close", + "process_symbol_in_dict", +): + setattr(_scicode_cmp, _name, globals()[_name]) +_scicode = _types.ModuleType("scicode") +_scicode_compare = _types.ModuleType("scicode.compare") +_scicode_compare.cmp = _scicode_cmp +_scicode.compare = _scicode_compare +_sys.modules.setdefault("scicode", _scicode) +_sys.modules.setdefault("scicode.compare", _scicode_compare) +_sys.modules["scicode.compare.cmp"] = _scicode_cmp +""" + + +def encode_targets(targets: list) -> str: + """Serialize h5 targets to a base64 string safe to embed in source.""" + return base64.b64encode(zlib.compress(pickle.dumps(targets))).decode("ascii") + + +def build_test_program(code_content: str, targets_b64: str, test_cases: list) -> str: + """Return a runnable program: solution code + shim + targets + test cases. + + *code_content* is ``dependencies + prior-step funcs + current-step func``. + *targets_b64* is :func:`encode_targets` output. *test_cases* are the raw + upstream test-body strings; each references ``target`` (the i-th target). + """ + parts = [ + code_content, + "", + "# --- sieval scicode test harness (injected) ---", + "import base64 as _b64, zlib as _zlib, pickle as _pkl", + _cmp_source(), + _SHIM_FOOTER, + f'targets = _pkl.loads(_zlib.decompress(_b64.b64decode("{targets_b64}")))', + "", + ] + for idx, case in enumerate(test_cases): + parts.append(f"target = targets[{idx}]\n") + parts.append(case) + parts.append("") + return "\n".join(parts) diff --git a/sieval/community/scicode/parse.py b/sieval/community/scicode/parse.py new file mode 100644 index 00000000..f2c06f6b --- /dev/null +++ b/sieval/community/scicode/parse.py @@ -0,0 +1,171 @@ +# Copyright 2024 The SciCode authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Code-extraction and HDF5 target readers adapted from SciCode @ 69a8cfc: +# - extract_function_name / get_function_from_code / process_hdf5_* : +# src/scicode/parse/parse.py +# - extract_python_script : src/scicode/gen/models.py +# Only the read-side helpers used by sieval are vendored; the h5 write-side +# (save_*_to_hdf5) and jsonl loaders are omitted. process_hdf5_to_tuple takes +# an explicit file path instead of the upstream module-level H5PY_FILE constant. +import ast +import re + + +def extract_function_name(function_header): + pattern = r"\bdef\s+(\w+)\s*\(" + match = re.search(pattern, function_header) + if match: + return match.group(1) + else: + pattern = r"\bclass\s+(\w+)\s*\(" + match = re.search(pattern, function_header) + if match: + return match.group(1) + else: + raise ValueError("Function name or class name not found.") + + +def get_function_from_code(code_string, function_name): + """Return the source of *function_name* extracted from *code_string*. + + Returns ``None`` if *code_string* is ``None``; on parse failure returns the + original string (upstream fallback behaviour). + """ + if code_string is None: + return None + try: + tree = ast.parse(code_string) + for node in ast.walk(tree): + if ( + isinstance(node, (ast.FunctionDef, ast.ClassDef)) + and node.name == function_name + ): + return ast.unparse(node) + except Exception as e: + print(f"{function_name} not found with error: {e}") + return code_string + + +def extract_python_script(response: str): + # Extract the python script from a model response and strip import lines + # (dependencies are prepended once from `required_dependencies`). + if "```" in response: + python_script = ( + response.split("```python")[1].split("```")[0] + if "```python" in response + else response.split("```")[1].split("```")[0] + ) + else: + print("Fail to extract python code from specific format.") + python_script = response + python_script = re.sub( + r"^\s*(import .*|from .*\s+import\s+.*)", "", python_script, flags=re.MULTILINE + ) + return python_script + + +def process_hdf5_list(group): + lst = [] + for key in group.keys(): + lst.append(group[key][()]) + return lst + + +def process_hdf5_dict(group): + import h5py # lazy: h5py is an optional (scicode-group) dependency + + dict = {} + for key, obj in group.items(): + if isinstance(obj, h5py.Group): + dict[key] = process_hdf5_sparse_matrix(obj["sparse_matrix"]) + elif isinstance(obj[()], bytes): + dict[key] = obj[()].decode("utf-8", errors="strict") + else: + try: + tmp = float(key) + dict[tmp] = obj[()] + except ValueError: + dict[key] = obj[()] + return dict + + +def process_hdf5_sparse_matrix(group): + import scipy.sparse # lazy: scipy is an optional (scicode-group) dependency + + data = group["data"][()] + shape = tuple(group["shape"][()]) + if "row" in group and "col" in group: + row = group["row"][()] + col = group["col"][()] + return scipy.sparse.coo_matrix((data, (row, col)), shape=shape) + elif "blocksize" in group: + indices = group["indices"][()] + indptr = group["indptr"][()] + blocksize = tuple(group["blocksize"][()]) + return scipy.sparse.bsr_matrix( + (data, indices, indptr), shape=shape, blocksize=blocksize + ) + else: + indices = group["indices"][()] + indptr = group["indptr"][()] + return scipy.sparse.csr_matrix((data, indices, indptr), shape=shape) + + +def process_hdf5_datagroup(group): + for key in group.keys(): + if key == "list": + return process_hdf5_list(group[key]) + if key == "sparse_matrix": + return process_hdf5_sparse_matrix(group[key]) + else: + return process_hdf5_dict(group) + + +def process_hdf5_to_tuple(step_id, test_num, h5py_file): + import h5py # lazy: h5py is an optional (scicode-group) dependency + + data_lst = [] + with h5py.File(h5py_file, "r") as f: + for test_id in range(test_num): + group_path = f"{step_id}/test{test_id + 1}" + if isinstance(f[group_path], h5py.Group): + group = f[group_path] # test1, test2, test3 + num_keys = [key for key in group.keys()] + if len(num_keys) == 1: # only 1 var in the test + subgroup = group[num_keys[0]] + if isinstance(subgroup, h5py.Dataset): + if isinstance(subgroup[()], bytes): + data_lst.append(subgroup[()].decode("utf-8", errors="strict")) + else: + data_lst.append(subgroup[()]) + elif isinstance(subgroup, h5py.Group): + data_lst.append(process_hdf5_datagroup(subgroup)) + else: + var_lst = [] + for key in group.keys(): # var1, var2, var3 + subgroup = group[key] + if isinstance(subgroup, h5py.Dataset): + if isinstance(subgroup[()], bytes): + var_lst.append( + subgroup[()].decode("utf-8", errors="strict") + ) + else: + var_lst.append(subgroup[()]) + elif isinstance(subgroup, h5py.Group): + var_lst.append(process_hdf5_datagroup(subgroup)) + data_lst.append(tuple(var_lst)) + else: + raise FileNotFoundError(f"Path {group_path} not found in the file.") + return data_lst diff --git a/sieval/community/scicode/prompts.py b/sieval/community/scicode/prompts.py new file mode 100644 index 00000000..393c8dfb --- /dev/null +++ b/sieval/community/scicode/prompts.py @@ -0,0 +1,179 @@ +# Copyright 2024 The SciCode authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prompt templates and step-assembly logic adapted from SciCode @ 69a8cfc: +# - templates: eval/data/{multistep_template.txt,background_comment_template.txt} +# - assembly: eval/scripts/gencode_json.py (Gencode.process_problem_steps / +# process_problem_code / generate_prompt_with_steps) +# The upstream Gencode class is decoupled here from file IO and model calls: +# these pure functions take the already-generated prior-step code as an argument +# so the sieval task can drive the sequential loop in-memory. The upstream +# template naming is preserved: "with background" selects the multistep template +# (scientist background injected); "without background" selects the +# background-comment template (the model must produce its own background). + +from importlib import resources + +# eval/data/multistep_template.txt (used when with_background=True) +MULTISTEP_TEMPLATE = """PROBLEM DESCRIPTION: +You will be provided with problem steps along with background knowledge necessary for solving the problem. Your task will be to develop a Python solution focused on the next step of the problem-solving process. + +PROBLEM STEPS AND FUNCTION CODE: +Here, you'll find the Python code for the initial steps of the problem-solving process. This code is integral to building the solution. + +{problem_steps_str} + +NEXT STEP - PROBLEM STEP AND FUNCTION HEADER: +This part will describe the next step in the problem-solving process. A function header will be provided, and your task is to develop the Python code for this next step based on the provided description and function header. + +{next_step_str} + +DEPENDENCIES: +Use only the following dependencies in your solution. Do not include these dependencies at the beginning of your code. + +{dependencies} + +RESPONSE GUIDELINES: +Now, based on the instructions and information provided above, write the complete and executable Python program for the next step in a single block. +Your response should focus exclusively on implementing the solution for the next step, adhering closely to the specified function header and the context provided by the initial steps. +Your response should NOT include the dependencies and functions of all previous steps. If your next step function calls functions from previous steps, please make sure it uses the headers provided without modification. +DO NOT generate EXAMPLE USAGE OR TEST CODE in your response. Please make sure your response python code in format of ```python```. +""" + +# eval/data/background_comment_template.txt (used when with_background=False) +BACKGROUND_COMMENT_TEMPLATE = """PROBLEM DESCRIPTION: +You will be provided with the main description of the problem, previous steps, and the next step. Your task will be to generate the disciplinary knowledge necessary for solving the next step and then develop a Python solution focused on this step. + +PREVIOUS STEPS DESCRIPTION: +{problem_steps_str} + +NEXT STEP - PROBLEM DESCRIPTION AND FUNCTION HEADER: +This part will describe the next step in the problem-solving process. First, provide the necessary scientific background knowledge as a comment at the beginning of your response, starting with 'Background: '. Then, a function header will be provided, and your task is to develop the Python code for this next step based on the provided description and function header. + +{next_step_str} + +DEPENDENCIES: +Use only the following dependencies in your solution. Do not include these dependencies at the beginning of your code. +{dependencies} + +RESPONSE GUIDELINES: +1. Start with the scientific background required for the next step, formatted as a comment. +2. Then write the complete and executable Python program for the next step in a single block. +3. Your response should focus exclusively on implementing the solution for the next step, adhering closely to the specified function header and the context provided by the initial steps. +4. DO NOT include previous function code, example usage or test code in your response. +5. Ensure your response is in the format of ```python``` and includes the necessary background as a comment at the top. + +Example: +```python +# Background: [Here, insert the necessary scientific knowledge required for the next step.] + +[Insert the Python code here based on the provided function header and dependencies.] +``` +""" + +# Steps that upstream does NOT generate; their scientist-authored gold code +# (eval/data/{prob}.{step}.txt) is used as context for later steps and the +# steps themselves are never tested. Keyed by (problem_id, zero-based step idx). +SPECIAL_STEPS = {("13", 5), ("62", 0), ("76", 2)} + + +def is_special_step(problem_id: str, step_idx: int) -> bool: + """True if (problem_id, zero-based step_idx) is a non-generated gold step.""" + return (str(problem_id), step_idx) in SPECIAL_STEPS + + +def special_step_code(step_number: str) -> str: + """Return the scientist-authored gold code for a special step. + + *step_number* is the ``sub_steps[i]["step_number"]`` value (e.g. ``"13.6"``), + which also names the vendored ``data/{step_number}.txt`` file. + """ + return ( + resources.files("sieval.community.scicode") + .joinpath("data", f"{step_number}.txt") + .read_text(encoding="utf-8") + ) + + +def prompt_template(with_background: bool) -> str: + return MULTISTEP_TEMPLATE if with_background else BACKGROUND_COMMENT_TEMPLATE + + +def _process_problem_code(sub_steps: list, num_steps: int) -> str: + header_docstring = sub_steps[num_steps - 1]["function_header"] + return_str = sub_steps[num_steps - 1]["return_line"] + return f"{header_docstring}\n\n{return_str}" + + +def process_problem_steps( + sub_steps: list, + num_steps: int, + previous_llm_code: list, + with_background: bool, +): + """Assemble prior-steps text, the next-step block, and the prior-code string. + + *previous_llm_code* holds the generated (import-stripped) code for each prior + step, indexed by zero-based step. Mirrors Gencode.process_problem_steps, + including the upstream operator precedence of the with_background ternary. + """ + output_lines = [] + previous_code = [] + for i in range(num_steps - 1): + output_lines.append( + sub_steps[i]["step_description_prompt"] + "\n" + sub_steps[i]["step_background"] + if with_background + else sub_steps[i]["step_description_prompt"] + ) + output_lines.append(previous_llm_code[i]) + previous_code.append(previous_llm_code[i]) + output_lines.append("------") + + next_step = [ + sub_steps[num_steps - 1]["step_description_prompt"] + + "\n" + + sub_steps[num_steps - 1]["step_background"] + if with_background + else sub_steps[num_steps - 1]["step_description_prompt"], + _process_problem_code(sub_steps, num_steps), + ] + output_str = "\n\n".join(output_lines[:-1]) # Remove the last "------" + next_step_str = "\n\n".join(next_step) + previous_code_str = "\n".join(previous_code) + return output_str, next_step_str, previous_code_str + + +def generate_prompt_with_steps( + sub_steps: list, + required_dependencies: str, + num_steps: int, + previous_llm_code: list, + with_background: bool, +): + """Return ``(prompt, previous_code)`` for the *num_steps*-th step (1-based). + + ``previous_code`` is ``f"{dependencies}\\n{previous_code_str}\\n"`` — the + import block plus accumulated prior functions, ready to prepend to the + newly generated step code before testing. + """ + problem_steps_str, next_step_str, previous_code_str = process_problem_steps( + sub_steps, num_steps, previous_llm_code, with_background + ) + assert next_step_str + prompt = prompt_template(with_background).format( + problem_steps_str=problem_steps_str, + next_step_str=next_step_str, + dependencies=required_dependencies, + ) + return prompt, f"{required_dependencies}\n{previous_code_str}\n" diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index 1973cbc5..8e00c072 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -105,6 +105,10 @@ from .openbookqa import ( OpenBookQADataset, OpenBookQADatasetSample, ) +from .scicode import ( + SciCodeDataset, + SciCodeDatasetSample, +) from .simpleqa_verified import ( SimpleQAVerifiedDataset, SimpleQAVerifiedDatasetSample, @@ -171,6 +175,8 @@ __all__ = [ "MMMLUDatasetSample", "OpenBookQADataset", "OpenBookQADatasetSample", + "SciCodeDataset", + "SciCodeDatasetSample", "SimpleQAVerifiedDataset", "SimpleQAVerifiedDatasetSample", "TEvalBeforeCallingDataset", diff --git a/sieval/datasets/scicode.py b/sieval/datasets/scicode.py new file mode 100644 index 00000000..a8f505b2 --- /dev/null +++ b/sieval/datasets/scicode.py @@ -0,0 +1,98 @@ +"""SciCode dataset — scientific research coding benchmark (80 problems / 338 steps). + +Problems come from the upstream GitHub jsonl (revision-pinned); the numeric +test-target file ``raw_ground.h5`` (byte-identical to the official +``test_data.h5``) is mirrored on HuggingFace as a plain HTTPS artifact, avoiding +the upstream Google-Drive dependency. The h5 path where the dataset was staged +is captured on :attr:`h5_path` for the task to read numeric targets from. + +AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic) +""" + +import os +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset, ensure_dataset_dict + +# github.com/scicode-bench/SciCode +SCICODE_GITHUB_REVISION = "69a8cfc829fe8788a426ce8b5de6292366dce7ef" +# huggingface.co/datasets/akshathmangudi/SciCode (raw/raw_ground.h5 == official +# test_data.h5; verified by matching sha256). +SCICODE_H5_MIRROR_REVISION = "526f6cbf273fc2b134444104fd1979a013e95d39" +H5_FILENAME = "raw_ground.h5" + + +class SciCodeSubStep(TypedDict): + step_number: str + step_description_prompt: str + step_background: str + ground_truth_code: str + function_header: str + return_line: str + test_cases: list[str] + + +class SciCodeDatasetSample(TypedDict): + problem_id: str + problem_name: str + required_dependencies: str + sub_steps: list[SciCodeSubStep] + + +@sieval_dataset( + name="scicode", + display_name="SciCode", + description="Research coding benchmark: 80 problems decomposed into 338 dependent sub-steps.", # noqa: E501 + source=( + f"url:https://raw.githubusercontent.com/scicode-bench/SciCode/{SCICODE_GITHUB_REVISION}/eval/data/problems_all.jsonl", + f"url:https://raw.githubusercontent.com/scicode-bench/SciCode/{SCICODE_GITHUB_REVISION}/eval/data/problems_dev.jsonl", + f"url:https://huggingface.co/datasets/akshathmangudi/SciCode/resolve/{SCICODE_H5_MIRROR_REVISION}/raw/{H5_FILENAME}", + ), + checksums={ + "problems_all.jsonl": "sha256:38797fef78f434720be6d053b4f3a86839d6f8ea5fb9115450677cd3a6edf81d", # noqa: E501 + "problems_dev.jsonl": "sha256:193968aff23b7ed931c8f6d196b611e2f6694adb49e6cc085444a60ad2fc4f7b", # noqa: E501 + "raw_ground.h5": "sha256:48b0272a88b17dbd29777c217e1b4fb2b019b92e11cc2add847409db9541b890", # noqa: E501 + }, + categories=(Category(Level1Category.CODE, "CodeGeneration"),), + tags=("english", "python", "code-exec"), + license="apache-2.0", +) +class SciCodeDataset(Dataset[SciCodeDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + # Capture the staged h5 location so the task can read numeric targets. + # copy.copy-based clones (slice/shuffle/...) preserve this attribute. + self._h5_path = os.path.join(name_or_path, H5_FILENAME) + + all_path = os.path.join(name_or_path, "problems_all.jsonl") + dev_path = os.path.join(name_or_path, "problems_dev.jsonl") + # Load each split independently: problems_dev carries extra optional + # columns (general_solution, sub_steps[].ground_truth_code) absent from + # problems_all, which trips HF's single-schema inference if the two are + # generated together. Splits may keep distinct columns; the task reads + # only the fields both share. + test = ensure_dataset( + load_dataset("json", data_files=all_path, split="train", **kwargs) + ) + dev = ensure_dataset( + load_dataset("json", data_files=dev_path, split="train", **kwargs) + ) + return ensure_dataset_dict(HFDatasetDict({"test": test, "dev": dev})) + + @property + def h5_path(self) -> str | None: + """Filesystem path to the staged numeric test-target h5. + + ``None`` when the dataset was built from a pre-loaded dict (e.g. tests) + rather than via :meth:`load`. + """ + return getattr(self, "_h5_path", None) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 1df783ed..436c4bb8 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -574,6 +574,34 @@ "license": "Apache-2.0", "checksums": {} }, + { + "name": "scicode", + "display_name": "SciCode", + "description": "Research coding benchmark: 80 problems decomposed into 338 dependent sub-steps.", + "source": [ + "url:https://raw.githubusercontent.com/scicode-bench/SciCode/69a8cfc829fe8788a426ce8b5de6292366dce7ef/eval/data/problems_all.jsonl", + "url:https://raw.githubusercontent.com/scicode-bench/SciCode/69a8cfc829fe8788a426ce8b5de6292366dce7ef/eval/data/problems_dev.jsonl", + "url:https://huggingface.co/datasets/akshathmangudi/SciCode/resolve/526f6cbf273fc2b134444104fd1979a013e95d39/raw/raw_ground.h5" + ], + "categories": [ + { + "level1": "Code", + "level2": "CodeGeneration" + } + ], + "tags": [ + "english", + "python", + "code-exec" + ], + "deps_group": null, + "license": "apache-2.0", + "checksums": { + "problems_all.jsonl": "sha256:38797fef78f434720be6d053b4f3a86839d6f8ea5fb9115450677cd3a6edf81d", + "problems_dev.jsonl": "sha256:193968aff23b7ed931c8f6d196b611e2f6694adb49e6cc085444a60ad2fc4f7b", + "raw_ground.h5": "sha256:48b0272a88b17dbd29777c217e1b4fb2b019b92e11cc2add847409db9541b890" + } + }, { "name": "simpleqa_verified", "display_name": "SimpleQA Verified", @@ -1313,6 +1341,27 @@ }, "status": "stable" }, + { + "name": "scicode_0shot_gen", + "display_name": "SciCode (0-shot, generative)", + "description": "SciCode — research coding benchmark with dependent sub-steps; sub-problem and main-problem accuracy.", + "dataset": "scicode", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "python", + "code-exec" + ], + "deps_group": "scicode", + "model_type": "chat", + "reference_impl": { + "source": "scicode", + "url": "https://github.com/scicode-bench/SciCode/tree/69a8cfc829fe8788a426ce8b5de6292366dce7ef/eval/scripts", + "notes": "Prompt assembly (gencode_json.py), code/h5 parsers, comparison helpers, and the 3 gold steps vendored into community/scicode. h5 targets are inlined into the sandbox program instead of read via process_hdf5_to_tuple in-sandbox." + }, + "status": "stable" + }, { "name": "simpleqa_verified_0shot_gen", "display_name": "SimpleQA Verified (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 96e1cd34..6c58d6aa 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -97,6 +97,9 @@ from .mmmlu_kshot_clp import ( from .openbookqa_kshot_gen import ( OpenBookQAFewShotGenTask, ) +from .scicode_0shot_gen import ( + SciCodeZeroShotGenTask, +) from .simpleqa_verified_0shot_gen import ( SimpleQAVerifiedZeroShotGenTask, ) @@ -140,6 +143,7 @@ __all__ = [ "MMLUZeroShotGenTask", "MMMLUKShotClpTask", "OpenBookQAFewShotGenTask", + "SciCodeZeroShotGenTask", "SimpleQAVerifiedZeroShotGenTask", "TEvalBeforeCallingZeroShotGenTask", "TheoremQAKShotBaseGenTask", diff --git a/sieval/tasks/scicode_0shot_gen.py b/sieval/tasks/scicode_0shot_gen.py new file mode 100644 index 00000000..3d35fcdb --- /dev/null +++ b/sieval/tasks/scicode_0shot_gen.py @@ -0,0 +1,323 @@ +"""SciCode 0-shot task for instruct/chat models. + +Each sample is one main problem decomposed into dependent sub-steps. Generation +is sequential *within* a problem: the prompt for step *i* embeds the model's own +code from steps ``1..i-1`` (upstream's default self-dependency setting, not gold +context). Different problems still run concurrently as separate samples. + +Evaluation mirrors upstream ``test_generated_code.py`` — per step, concatenate +``required_dependencies`` + prior-step functions + current-step function + the +step's test cases, then execute — but the numeric targets that upstream reads +from ``test_data.h5`` inside the sandbox are read here on the eval side and +inlined into the program, keeping sieval's code-eval sandbox stateless. Three +scientist-authored steps (13.6, 62.1, 76.3) are not generated or tested; their +gold code is used only as context for later steps. + +Metrics: sub-problem accuracy (passing steps / tested steps) and main-problem +accuracy (problems whose every tested step passes) — the headline resolve rate. + +AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic) +""" + +import os +import time +from typing import TypedDict, override + +import httpx +from loguru import logger +from openai.types.chat import ChatCompletionUserMessageParam + +from sieval.community.scicode import ( + build_test_program, + encode_targets, + extract_python_script, + generate_prompt_with_steps, + is_special_step, + process_hdf5_to_tuple, + special_step_code, +) +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + ReferenceImpl, + Task, + TaskStageOutput, + sieval_task, +) +from sieval.core.utils.meta import build_stage_meta +from sieval.datasets import SciCodeDatasetSample + + +class StepCode(TypedDict): + step_number: str + tested: bool + # dependencies + prior-step funcs + current-step func; None for special steps + code_content: str | None + # Raw model response for this step, kept for provenance/debugging. Empty for + # special (gold) steps that are not generated. + raw_response: str + # True when extract_python_script found no code in raw_response — the step + # (and any later step depending on it) will fail; surfaced in the report. + empty_extraction: bool + + +class StepProgram(TypedDict): + step_number: str + program: str + empty_extraction: bool + + +class StepFeedback(TypedDict): + step_number: str + correct: bool + msg: str + empty_extraction: bool + + +@sieval_task( + name="scicode_0shot_gen", + display_name="SciCode (0-shot, generative)", + description="SciCode — research coding benchmark with dependent sub-steps; sub-problem and main-problem accuracy.", # noqa: E501 + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "python", "code-exec"), + model_type="chat", + deps_group="scicode", + reference_impl=ReferenceImpl( + source="scicode", + url="https://github.com/scicode-bench/SciCode/tree/69a8cfc829fe8788a426ce8b5de6292366dce7ef/eval/scripts", + notes=( + "Prompt assembly (gencode_json.py), code/h5 parsers, comparison " + "helpers, and the 3 gold steps vendored into community/scicode. " + "h5 targets are inlined into the sandbox program instead of read via " + "process_hdf5_to_tuple in-sandbox." + ), + ), +) +class SciCodeZeroShotGenTask( + Task[ + SciCodeDatasetSample, + SciCodeDatasetSample, + TaskStageOutput[list[StepCode]], + list[StepProgram], + list[StepFeedback], + dict[str, float], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + with_background: bool = False, + h5_path: str | None = None, + max_concurrency: int = 4, + timeout: float = 300.0, + ): + super().__init__(dataset=dataset, model=model, name=name) + self._with_background = with_background + self._h5_path = h5_path + self._timeout = timeout + self._code_eval_api = os.getenv( + "SIEVAL_CODE_EVAL_API", "http://localhost:11451/evaluations" + ) + self._http_client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=max_concurrency) + ) + + @override + async def setup(self): + if self._h5_path is None: + self._h5_path = getattr(self.dataset, "h5_path", None) + if not self._h5_path or not os.path.exists(self._h5_path): + raise FileNotFoundError( + "SciCode numeric test data not found. Run " + "`sieval dataset download scicode` to stage raw_ground.h5, or pass " + f"h5_path=. Resolved path: {self._h5_path!r}" + ) + + @override + async def preprocess(self, raw, ctx): + # Prompt assembly needs the model's own prior-step code, so it happens in + # infer; preprocess passes the problem through unchanged. + return raw + + @override + async def infer(self, pre, ctx): + sub_steps = pre["sub_steps"] + problem_id = str(pre["problem_id"]) + deps = pre["required_dependencies"] + tot = len(sub_steps) + + previous_llm_code: list[str | None] = [None] * tot + steps_out: list[StepCode] = [] + outputs: list[ModelOutput] = [] + + for i in range(tot): + step = sub_steps[i] + step_number = step["step_number"] + + if is_special_step(problem_id, i): + # Scientist-authored gold code: context only, never generated/tested. + # Embed the whole gold file verbatim — do NOT extract a single + # function by name. These files are self-contained, import-free + # blocks that may define a class (13.6 -> Maxwell) or several + # top-level objects (62.1 -> Block AND EnlargedBlock); pulling one + # node by func_name (which resolves to `__init__` for a class + # header) would drop the class and break every later step that + # references it. + previous_llm_code[i] = special_step_code(step_number) + steps_out.append( + { + "step_number": step_number, + "tested": False, + "code_content": None, + "raw_response": "", + "empty_extraction": False, + } + ) + continue + + prompt, previous_code = generate_prompt_with_steps( + sub_steps, deps, i + 1, previous_llm_code, self._with_background + ) + messages: list[ChatCompletionUserMessageParam] = [ + {"role": "user", "content": prompt} + ] + output = await self.model.agenerate(messages, n=1) + outputs.append(output) + raw_response = output.texts[0] + extracted = extract_python_script(raw_response) + # An extraction with no def/class means the model returned prose, was + # truncated, or emitted an unfenced answer; the step will fail and so + # will any later step calling its function. Log it and flag for the + # report instead of silently scoring 0. + empty = "def " not in extracted and "class " not in extracted + if empty: + logger.warning( + "SciCode empty code extraction: problem {} step {} " + "(finish={}, raw_len={}); dependent steps may cascade-fail.", + problem_id, + step_number, + (output.finish_reasons or ["?"])[0], + len(raw_response), + ) + previous_llm_code[i] = extracted + # Matches upstream save_response_with_steps: `{previous_code}\n{code}`. + steps_out.append( + { + "step_number": step_number, + "tested": True, + "code_content": f"{previous_code}\n{extracted}", + "raw_response": raw_response, + "empty_extraction": empty, + } + ) + + # Box the structured per-step code as the stage value while recording + # token usage from every model call via the stage meta. + return TaskStageOutput(value=steps_out, meta=build_stage_meta(*outputs)) + + @override + async def postprocess(self, inf, ctx): + steps_out: list[StepCode] = inf.value + sub_steps = ctx.raw_sample["sub_steps"] + by_number = {s["step_number"]: s for s in sub_steps} + + programs: list[StepProgram] = [] + for sc in steps_out: + if not sc["tested"] or sc["code_content"] is None: + continue + step_number = sc["step_number"] + test_cases = by_number[step_number]["test_cases"] + targets = process_hdf5_to_tuple(step_number, len(test_cases), self._h5_path) + program = build_test_program( + sc["code_content"], encode_targets(targets), test_cases + ) + programs.append( + { + "step_number": step_number, + "program": program, + "empty_extraction": sc["empty_extraction"], + } + ) + return programs + + @override + async def feedback(self, post, ctx): + feedbacks: list[StepFeedback] = [] + for step in post: + try: + resp = await self._http_client.post( + self._code_eval_api, + json={ + "uuid": f"{step['step_number']}-{time.perf_counter_ns()}", + "source": "scicode", + "code": step["program"], + "timeout": self._timeout, + }, + timeout=self._timeout + 5, # buffer for network latency + ) + resp.raise_for_status() + res = resp.json() + feedbacks.append( + { + "step_number": step["step_number"], + "correct": res["status"], + "msg": res["msg"], + "empty_extraction": step["empty_extraction"], + } + ) + except Exception as e: + logger.warning( + "SciCode eval error for step {}: [{}] {}", + step["step_number"], + type(e).__name__, + e, + ) + raise e + return True, feedbacks + + @override + async def report(self, finals, fails): + total_problems = len(finals) + len(fails) + if total_problems == 0: + return {"score": 0.0, "fails": len(fails)} + + correct_steps = 0 + total_steps = 0 + correct_problems = 0 + empty_extractions = 0 + for f in finals: + feedbacks = f.feedback_result + n_correct = sum(1 for fb in feedbacks if fb["correct"]) + correct_steps += n_correct + total_steps += len(feedbacks) + empty_extractions += sum( + 1 for fb in feedbacks if fb.get("empty_extraction") + ) + if feedbacks and n_correct == len(feedbacks): + correct_problems += 1 + + # Pipeline failures (fails) count as unsolved problems; their step counts + # are unknown, so sub-problem accuracy is over evaluated steps only. + main_accuracy = correct_problems * 100 / total_problems + sub_accuracy = correct_steps * 100 / total_steps if total_steps else 0.0 + return { + "score": main_accuracy, + "main_problem_accuracy": main_accuracy, + "sub_problem_accuracy": sub_accuracy, + "correct_problems": correct_problems, + "total_problems": total_problems, + "correct_steps": correct_steps, + "total_steps": total_steps, + # Steps where the model produced no extractable code (truncation / + # unfenced / prose). A non-zero count means some failures are + # generation-side, not solution-correctness — investigate raw_response. + "empty_extractions": empty_extractions, + "fails": len(fails), + } + + @override + async def shutdown(self): + await self._http_client.aclose() diff --git a/tests/unit/datasets/test_scicode.py b/tests/unit/datasets/test_scicode.py new file mode 100644 index 00000000..0656baab --- /dev/null +++ b/tests/unit/datasets/test_scicode.py @@ -0,0 +1,87 @@ +"""Unit tests for the SciCode dataset loader. + +AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic) +""" + +import json + +from sieval.datasets.scicode import SciCodeDataset + + +def _problem(problem_id: str) -> dict: + return { + "problem_id": problem_id, + "problem_name": f"problem {problem_id}", + "required_dependencies": "import numpy as np", + "sub_steps": [ + { + "step_number": f"{problem_id}.1", + "step_description_prompt": "do step 1", + "step_background": "bg 1", + "ground_truth_code": "def f():\n return 1", + "function_header": "def f():", + "return_line": " return 1", + "test_cases": ["assert f() == 1"], + } + ], + } + + +def _dev_problem(problem_id: str) -> dict: + # The real problems_dev.jsonl carries extra columns absent from + # problems_all.jsonl (general_solution at top level, ground_truth_code per + # sub-step); the loader must tolerate the per-split schema difference. + p = _problem(problem_id) + p["general_solution"] = "def solve():\n return 1" + p["sub_steps"][0]["ground_truth_code"] = "def f():\n return 1" + return p + + +def _stage(tmp_path) -> str: + root = tmp_path / "scicode" + root.mkdir() + (root / "problems_all.jsonl").write_text( + "\n".join(json.dumps(_problem(str(i))) for i in (1, 2, 3)), encoding="utf-8" + ) + (root / "problems_dev.jsonl").write_text( + json.dumps(_dev_problem("1")), encoding="utf-8" + ) + # load() never reads the h5; a placeholder is enough to resolve the path. + (root / "raw_ground.h5").write_bytes(b"") + return str(root) + + +def test_load_builds_test_and_dev_splits(tmp_path): + ds = SciCodeDataset(_stage(tmp_path)) + assert set(ds.dataset_dict) == {"test", "dev"} + assert ds.test_set is not None + assert len(ds.test_set) == 3 + assert len(ds.dataset_dict["dev"]) == 1 + + row = ds.test_set[0] + assert row["problem_id"] == "1" + assert row["required_dependencies"] == "import numpy as np" + assert row["sub_steps"][0]["step_number"] == "1.1" + assert row["sub_steps"][0]["test_cases"] == ["assert f() == 1"] + + +def test_h5_path_points_at_staged_file(tmp_path): + root = _stage(tmp_path) + ds = SciCodeDataset(root) + assert ds.h5_path == f"{root}/raw_ground.h5" + + +def test_h5_path_survives_clone(tmp_path): + # slice() returns a copy.copy clone; the captured h5 path must persist so the + # task can still locate the numeric targets after sampling. + ds = SciCodeDataset(_stage(tmp_path)) + sliced = ds.slice(2) + assert sliced.test_set is not None + assert len(sliced.test_set) == 2 + assert sliced.h5_path == ds.h5_path + + +def test_h5_path_none_when_built_from_dict(tmp_path): + ds = SciCodeDataset(_stage(tmp_path)) + rebuilt = type(ds)(_hf_dict=ds.dataset_dict) + assert rebuilt.h5_path is None diff --git a/tests/unit/tasks/test_scicode_0shot_gen.py b/tests/unit/tasks/test_scicode_0shot_gen.py new file mode 100644 index 00000000..b1954b8e --- /dev/null +++ b/tests/unit/tasks/test_scicode_0shot_gen.py @@ -0,0 +1,310 @@ +"""Unit tests for the SciCode 0-shot generative task. + +AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic) +""" + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import TaskContext, TaskStageOutput +from sieval.datasets.scicode import SciCodeDataset +from sieval.tasks.scicode_0shot_gen import SciCodeZeroShotGenTask + +# h5py is an optional (scicode-group) dependency; skip the whole module without it. +h5py = pytest.importorskip("h5py") + + +class _ScriptedChatModel(ChatModel): + """Returns queued replies in order, recording each prompt and kwargs.""" + + def __init__(self, replies: list[str], model: str = "candidate"): + super().__init__(model=model, api_key="fake") + self._replies = list(replies) + self.prompts: list = [] + self.last_kwargs: dict[str, object] = {} + self.calls = 0 + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + self.prompts.append(prompt) + self.last_kwargs = dict(kwargs) + self.calls += 1 + return ModelOutput(model=self.meta(), texts=[self._replies.pop(0)]) + + async def _alogprobs_impl(self, prompt, **kwargs) -> ModelOutput: + _ = (prompt, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _code_reply(fn: str) -> str: + return f"```python\ndef {fn}():\n return 1\n```" + + +def _dataset() -> SciCodeDataset: + row = { + "problem_id": "1", + "problem_name": "p", + "required_dependencies": "import numpy as np", + "sub_steps": [], + } + return SciCodeDataset(_hf_dict=HFDatasetDict({"test": HFDataset.from_list([row])})) + + +def _substep(number: str, header: str, tests: list[str]) -> dict: + return { + "step_number": number, + "step_description_prompt": f"describe {number}", + "step_background": f"background {number}", + "ground_truth_code": "", + "function_header": header, + "return_line": " return 1", + "test_cases": tests, + } + + +def _task(model, **kw) -> SciCodeZeroShotGenTask: + kw.setdefault("h5_path", "unused") + return SciCodeZeroShotGenTask(_dataset(), model, **kw) + + +# --- infer: sequential, single-n, prior code fed forward --- + + +@pytest.mark.anyio +async def test_infer_is_sequential_and_feeds_prior_code(): + sub_steps = [ + _substep("1.1", "def step_a():", ["assert step_a() == 1"]), + _substep("1.2", "def step_b():", ["assert step_b() == 1"]), + ] + model = _ScriptedChatModel([_code_reply("step_a"), _code_reply("step_b")]) + task = _task(model) + raw = { + "problem_id": "1", + "required_dependencies": "import numpy as np", + "sub_steps": sub_steps, + } + + boxed = await task.infer(raw, TaskContext(sample_id=0, raw_sample=raw)) + + assert isinstance(boxed, TaskStageOutput) + assert model.calls == 2 + # Only the scheduling knob is forwarded to the model layer. + assert model.last_kwargs == {"n": 1} + # The second prompt must embed the first step's generated function. + assert "def step_a" in model.prompts[1][0]["content"] + + steps = boxed.value + assert [s["tested"] for s in steps] == [True, True] + # code_content = dependencies + accumulated prior funcs + current func. + assert steps[1]["code_content"].startswith("import numpy as np") + assert "def step_a" in steps[1]["code_content"] + assert "def step_b" in steps[1]["code_content"] + + +@pytest.mark.anyio +async def test_infer_uses_gold_context_for_special_step_without_calling_model(): + # Problem 76 step 3 (index 2) is scientist-authored: never generated/tested, + # its gold code is fed as context to later steps. + sub_steps = [ + _substep("76.1", "def step_a():", ["assert step_a() == 1"]), + _substep("76.2", "def step_b():", ["assert step_b() == 1"]), + _substep("76.3", "def generate_dna(N, PWM):", ["assert True"]), + _substep("76.4", "def step_d():", ["assert step_d() == 1"]), + ] + model = _ScriptedChatModel( + [_code_reply("step_a"), _code_reply("step_b"), _code_reply("step_d")] + ) + task = _task(model) + raw = { + "problem_id": "76", + "required_dependencies": "import numpy as np", + "sub_steps": sub_steps, + } + + boxed = await task.infer(raw, TaskContext(sample_id=0, raw_sample=raw)) + steps = boxed.value + + # 3 generated steps -> 3 model calls; the special step is skipped. + assert model.calls == 3 + assert steps[2]["tested"] is False + assert steps[2]["code_content"] is None + # The 4th step's prompt must contain the gold generate_dna function. + assert "def generate_dna" in model.prompts[2][0]["content"] + + +@pytest.mark.anyio +async def test_infer_embeds_full_gold_class_for_class_special_step(): + # Problem 62 step 1 (index 0) is a gold step whose file defines TWO classes + # (Block AND EnlargedBlock) under a `class ...:` header. The gold must be + # embedded verbatim: extracting a single node by the header's function name + # (which resolves to `__init__` for a class header) would drop the `class` + # wrapper and the second class, breaking every later step that uses them. + sub_steps = [ + _substep( + "62.1", "class EnlargedBlock:\n def __init__(self):", ["assert True"] + ), + _substep("62.2", "def uses_block():", ["assert uses_block() == 1"]), + ] + model = _ScriptedChatModel([_code_reply("uses_block")]) + task = _task(model) + raw = { + "problem_id": "62", + "required_dependencies": "import numpy as np", + "sub_steps": sub_steps, + } + + await task.infer(raw, TaskContext(sample_id=0, raw_sample=raw)) + + # Only the one non-special step calls the model. + assert model.calls == 1 + # The gold context injected into step 62.2's prompt must contain BOTH classes + # with their `class` keyword intact (not just an unwrapped __init__). + gold_ctx = model.prompts[0][0]["content"] + assert "class Block" in gold_ctx + assert "class EnlargedBlock" in gold_ctx + + +@pytest.mark.anyio +async def test_infer_flags_empty_extraction_when_model_returns_no_code(): + # A prose-only / truncated response yields no def/class: the step must be + # flagged so the report can distinguish generation failure from wrong answers. + sub_steps = [_substep("5.1", "def only_step():", ["assert only_step() == 1"])] + model = _ScriptedChatModel(["I cannot help with that."]) # no code fence + task = _task(model) + raw = { + "problem_id": "5", + "required_dependencies": "import numpy as np", + "sub_steps": sub_steps, + } + + boxed = await task.infer(raw, TaskContext(sample_id=0, raw_sample=raw)) + step = boxed.value[0] + assert step["empty_extraction"] is True + assert step["raw_response"] == "I cannot help with that." + # A normal fenced reply must NOT be flagged (discriminates the check). + model2 = _ScriptedChatModel([_code_reply("only_step")]) + boxed2 = await _task(model2).infer(raw, TaskContext(sample_id=0, raw_sample=raw)) + assert boxed2.value[0]["empty_extraction"] is False + + +# --- postprocess: h5 targets inlined into a self-contained, runnable program --- + + +def _write_h5(path, step_number: str, value): + with h5py.File(path, "w") as f: + f.create_dataset(f"{step_number}/test1/var1", data=value) + + +@pytest.mark.anyio +async def test_postprocess_builds_program_that_executes(tmp_path): + # The injected comparison shim (vendored cmp) imports sympy; it runs in the + # code-eval sandbox at runtime, not on the eval side, so sympy is not a + # declared eval-side dep. Exercise the full program only where it's present. + pytest.importorskip("sympy") + h5 = tmp_path / "raw_ground.h5" + _write_h5(h5, "9.1", 42) + model = _ScriptedChatModel([]) + task = _task(model, h5_path=str(h5)) + + sub_step = _substep("9.1", "def f():", ["assert np.allclose(f(), target)"]) + raw = {"problem_id": "9", "sub_steps": [sub_step]} + inf = TaskStageOutput( + value=[ + { + "step_number": "9.1", + "tested": True, + "code_content": "import numpy as np\n\ndef f():\n return 42\n", + "raw_response": "```python\ndef f():\n return 42\n```", + "empty_extraction": False, + } + ] + ) + + programs = await task.postprocess(inf, TaskContext(sample_id=0, raw_sample=raw)) + assert len(programs) == 1 + program = programs[0]["program"] + + # The vendored comparison import and the correct target must both resolve at + # runtime; a matching solution passes without raising. + exec(compile(program, "", "exec"), {}) + + # A wrong solution must fail the injected assertion. + bad = program.replace("return 42", "return 0") + with pytest.raises(AssertionError): + exec(compile(bad, "", "exec"), {}) + + +# --- report: sub-problem and main-problem accuracy --- + + +def _final(sample_id, correct_flags, empty_flags=None): + empty_flags = empty_flags or [False] * len(correct_flags) + return TaskContext( + sample_id=sample_id, + feedback_result=[ + {"step_number": f"s{i}", "correct": c, "msg": "", "empty_extraction": e} + for i, (c, e) in enumerate(zip(correct_flags, empty_flags, strict=True)) + ], + ) + + +@pytest.mark.anyio +async def test_report_sub_and_main_accuracy(): + model = _ScriptedChatModel([]) + task = _task(model) + finals = [ + _final(0, [True, True]), # fully solved + _final(1, [True, False, True]), # partial + ] + report = await task.report(finals, fails=[]) + + # steps: 4 correct / 5 total = 80%; problems: 1 fully solved / 2 = 50%. + assert report["correct_steps"] == 4 + assert report["total_steps"] == 5 + assert report["sub_problem_accuracy"] == pytest.approx(80.0) + assert report["correct_problems"] == 1 + assert report["total_problems"] == 2 + assert report["main_problem_accuracy"] == pytest.approx(50.0) + assert report["score"] == report["main_problem_accuracy"] + + +@pytest.mark.anyio +async def test_report_counts_empty_extractions(): + model = _ScriptedChatModel([]) + task = _task(model) + # Sample 1 has one step whose code failed to extract (also counts as wrong). + finals = [ + _final(0, [True, True]), + _final(1, [False, True], empty_flags=[True, False]), + ] + report = await task.report(finals, fails=[]) + assert report["empty_extractions"] == 1 + # It still counts as an incorrect step, not a separate bucket. + assert report["correct_steps"] == 3 + assert report["total_steps"] == 4 + + +@pytest.mark.anyio +async def test_report_counts_pipeline_fails_as_unsolved_problems(): + model = _ScriptedChatModel([]) + task = _task(model) + finals = [_final(0, [True, True])] + fails = [TaskContext(sample_id=1)] + report = await task.report(finals, fails) + + # The failed problem dilutes main-problem accuracy: 1 solved / 2 = 50%. + assert report["fails"] == 1 + assert report["total_problems"] == 2 + assert report["main_problem_accuracy"] == pytest.approx(50.0) + # Sub accuracy is over evaluated steps only (fail step-count unknown). + assert report["total_steps"] == 2 + + +@pytest.mark.anyio +async def test_report_empty_is_zero(): + model = _ScriptedChatModel([]) + task = _task(model) + report = await task.report([], []) + assert report["score"] == 0.0 From 8b23289d2c3fdd69f3d21bb82d499be7108ea0b6 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Fri, 24 Jul 2026 14:33:17 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(scicode):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20inline=20gold=20steps,=20timeout=201800,=20doc=20de?= =?UTF-8?q?viations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sieval/community/scicode/__init__.py | 4 - sieval/community/scicode/_gold_steps.py | 129 ++++++++++++++++++++++++ sieval/community/scicode/harness.py | 2 + sieval/community/scicode/prompts.py | 12 +-- sieval/meta/index.json | 2 +- sieval/tasks/scicode_0shot_gen.py | 20 +++- 6 files changed, 151 insertions(+), 18 deletions(-) create mode 100644 sieval/community/scicode/_gold_steps.py diff --git a/sieval/community/scicode/__init__.py b/sieval/community/scicode/__init__.py index 7cf35781..60621e78 100644 --- a/sieval/community/scicode/__init__.py +++ b/sieval/community/scicode/__init__.py @@ -25,9 +25,7 @@ encode_targets, ) from sieval.community.scicode.parse import ( - extract_function_name, extract_python_script, - get_function_from_code, process_hdf5_to_tuple, ) from sieval.community.scicode.prompts import ( @@ -39,9 +37,7 @@ __all__ = [ "build_test_program", "encode_targets", - "extract_function_name", "extract_python_script", - "get_function_from_code", "process_hdf5_to_tuple", "generate_prompt_with_steps", "is_special_step", diff --git a/sieval/community/scicode/_gold_steps.py b/sieval/community/scicode/_gold_steps.py new file mode 100644 index 00000000..00039c1b --- /dev/null +++ b/sieval/community/scicode/_gold_steps.py @@ -0,0 +1,129 @@ +# Copyright 2024 The SciCode authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Scientist-authored gold code for the three SciCode steps upstream does NOT +generate (13.6, 62.1, 76.3); embedded verbatim as context for later steps. + +Vendored from SciCode @ 69a8cfc (eval/data/{prob}.{step}.txt) and inlined here so +they are committed and wheel-packaged (the repo-wide ``data/`` dir is gitignored). +""" + +GOLD_STEP_CODE: dict[str, str] = { + "13.6": r'''# code 1.6 +class Maxwell: + """ The base class for evolution of Maxwell's equations. + """ + + def __init__(self, n_grid, x_out): + """Constructor sets up coordinates, memory for variables. + The variables: + mesh points: + x: the x coordinate for each mesh grid + y: the y coordinate for each mesh grid + z: the z coordinate for each mesh grid + t: the time coordinate of the simulation + r: the distance to the origin for each mesh grid + evolving fields: + E_x: the x component of the field E + E_y: the y componnet of the field E + E_z: the z component of the field E + A_x: the x component of the field A + A_y: the y component of the field A + A_z: the z component of the field A + phi: the scalar potential field phi values + monitor variables: + constraint: the current constraint violation value from the evolving fields. + + """ + + self.n_grid = n_grid + self.n_vars = 7 + self.delta = float(x_out) / (n_grid - 2.0) + delta = self.delta + + self.x = np.linspace(-self.delta*0.5, x_out + 0.5*self.delta, self.n_grid)[:,None,None] + self.y = np.linspace(-self.delta*0.5, x_out + 0.5*self.delta, self.n_grid)[None,:,None] + self.z = np.linspace(-self.delta*0.5, x_out + 0.5*self.delta, self.n_grid)[None,None,:] + self.r = np.sqrt(self.x**2+self.y**2+self.z**2) + + + # set up all variables common to both approaches + self.E_x = zeros((n_grid, n_grid, n_grid)) + self.E_y = zeros((n_grid, n_grid, n_grid)) + self.E_z = zeros((n_grid, n_grid, n_grid)) + self.A_x = zeros((n_grid, n_grid, n_grid)) + self.A_y = zeros((n_grid, n_grid, n_grid)) + self.A_z = zeros((n_grid, n_grid, n_grid)) + self.phi = zeros((n_grid, n_grid, n_grid)) + self.constraint = zeros((n_grid, n_grid, n_grid)) + + + self.t = 0.0''', + "62.1": r'''class Block: + def __init__(self, length, basis_size, operator_dict): + self.length = length + self.basis_size = basis_size + self.operator_dict = operator_dict + + def print_all(self): + print(self.length) + print(self.basis_size) + for key, matrix in self.operator_dict.items(): + if isinstance(matrix, np.ndarray): + print(f"{key}:\n{matrix}\n") + else: + print(f"{key}:\n{matrix.toarray()}\n") + +class EnlargedBlock: + def __init__(self, length, basis_size, operator_dict): + self.length = length + self.basis_size = basis_size + self.operator_dict = operator_dict + + def print_all(self): + print(self.length) + print(self.basis_size) + for key, matrix in self.operator_dict.items(): + if isinstance(matrix, np.ndarray): + print(f"{key}:\n{matrix}\n") + else: + print(f"{key}:\n{matrix.toarray()}\n")''', + "76.3": r"""def generate_dna(N: int, PWM: dict) -> tuple: + ''' + Input: + N (int): Length of the resultant DNA sequence. + PWM matrix with keys 'A', 'C', 'G', 'T' + + Output: + tuple: Insertion location (int), DNA sequence (str), DNA reverse complement (str) + ''' + p = random.randint(0, N-1) + + nucleotide = "ACGT" + uni_weights = [0.25,0.25,0.25,0.25] #uniform distribution + dna_string = ''.join(random.choices(nucleotide, uni_weights, k=N)) + + spike_mat = load_motif_from_df(PWM) + spiked_seq = ''.join(random.choices(nucleotide, weights=[PWM[nuc][i] for nuc in nucleotide], k=1)[0] + for i in range(len(PWM['A']))) + + complement = {'A':'T', 'T':'A', 'C':'G', 'G':'C'} + reversed_seq = dna_string[::-1] + reverse_complement = ''.join(complement[nuc] for nuc in reversed_seq if nuc in complement) + + new_seq = dna_string[:p] + spiked_seq + dna_string[p:] + new_seq_rc = reverse_complement[:N-p] + spiked_seq + reverse_complement[N-p:] + + return p, new_seq, new_seq_rc""", +} diff --git a/sieval/community/scicode/harness.py b/sieval/community/scicode/harness.py index 468577e6..bd4fcb60 100644 --- a/sieval/community/scicode/harness.py +++ b/sieval/community/scicode/harness.py @@ -18,11 +18,13 @@ AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic) """ import base64 +import functools import pickle import zlib from importlib import resources +@functools.lru_cache(maxsize=1) def _cmp_source() -> str: return ( resources.files("sieval.community.scicode") diff --git a/sieval/community/scicode/prompts.py b/sieval/community/scicode/prompts.py index 393c8dfb..fb196b3b 100644 --- a/sieval/community/scicode/prompts.py +++ b/sieval/community/scicode/prompts.py @@ -23,7 +23,7 @@ # (scientist background injected); "without background" selects the # background-comment template (the model must produce its own background). -from importlib import resources +from ._gold_steps import GOLD_STEP_CODE # eval/data/multistep_template.txt (used when with_background=True) MULTISTEP_TEMPLATE = """PROBLEM DESCRIPTION: @@ -96,14 +96,10 @@ def is_special_step(problem_id: str, step_idx: int) -> bool: def special_step_code(step_number: str) -> str: """Return the scientist-authored gold code for a special step. - *step_number* is the ``sub_steps[i]["step_number"]`` value (e.g. ``"13.6"``), - which also names the vendored ``data/{step_number}.txt`` file. + *step_number* is the ``sub_steps[i]["step_number"]`` value (e.g. ``"13.6"``); + the gold code is inlined verbatim in :data:`GOLD_STEP_CODE`. """ - return ( - resources.files("sieval.community.scicode") - .joinpath("data", f"{step_number}.txt") - .read_text(encoding="utf-8") - ) + return GOLD_STEP_CODE[step_number] def prompt_template(with_background: bool) -> str: diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 436c4bb8..b6485b35 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1358,7 +1358,7 @@ "reference_impl": { "source": "scicode", "url": "https://github.com/scicode-bench/SciCode/tree/69a8cfc829fe8788a426ce8b5de6292366dce7ef/eval/scripts", - "notes": "Prompt assembly (gencode_json.py), code/h5 parsers, comparison helpers, and the 3 gold steps vendored into community/scicode. h5 targets are inlined into the sandbox program instead of read via process_hdf5_to_tuple in-sandbox." + "notes": "Vendored from upstream eval/scripts into community/scicode: prompt templates, code/h5 parsers, comparison helpers, and the 3 non-generated gold steps (13.6/62.1/76.3, embedded verbatim as context; inlined in _gold_steps.py). Deviations from upstream: (1) numeric h5 targets are read eval-side and inlined into the sandbox program (upstream reads test_data.h5 in-subprocess); (2) execution runs on a remote code-eval service over HTTP, not an in-process subprocess; (3) pipeline failures count as unsolved problems in main-problem accuracy. Reproducing official numbers requires greedy decoding (temperature=0) in the model config; with_background defaults to False (the official headline mode). The code-eval service image must provide the sandbox scientific stack including sympy, which the vendored comparison shim injects — see the evaluator's requirements/scicode.txt." }, "status": "stable" }, diff --git a/sieval/tasks/scicode_0shot_gen.py b/sieval/tasks/scicode_0shot_gen.py index 3d35fcdb..b6f923d5 100644 --- a/sieval/tasks/scicode_0shot_gen.py +++ b/sieval/tasks/scicode_0shot_gen.py @@ -87,10 +87,20 @@ class StepFeedback(TypedDict): source="scicode", url="https://github.com/scicode-bench/SciCode/tree/69a8cfc829fe8788a426ce8b5de6292366dce7ef/eval/scripts", notes=( - "Prompt assembly (gencode_json.py), code/h5 parsers, comparison " - "helpers, and the 3 gold steps vendored into community/scicode. " - "h5 targets are inlined into the sandbox program instead of read via " - "process_hdf5_to_tuple in-sandbox." + "Vendored from upstream eval/scripts into community/scicode: prompt " + "templates, code/h5 parsers, comparison helpers, and the 3 " + "non-generated gold steps (13.6/62.1/76.3, embedded verbatim as " + "context; inlined in _gold_steps.py). Deviations from upstream: " + "(1) numeric h5 targets are read eval-side and inlined into the " + "sandbox program (upstream reads test_data.h5 in-subprocess); " + "(2) execution runs on a remote code-eval service over HTTP, not an " + "in-process subprocess; (3) pipeline failures count as unsolved " + "problems in main-problem accuracy. Reproducing official numbers " + "requires greedy decoding (temperature=0) in the model config; " + "with_background defaults to False (the official headline mode). The " + "code-eval service image must provide the sandbox scientific stack " + "including sympy, which the vendored comparison shim injects — see " + "the evaluator's requirements/scicode.txt." ), ), ) @@ -112,7 +122,7 @@ def __init__( with_background: bool = False, h5_path: str | None = None, max_concurrency: int = 4, - timeout: float = 300.0, + timeout: float = 1800.0, # matches upstream test_generated_code.py ): super().__init__(dataset=dataset, model=model, name=name) self._with_background = with_background