feat(scicode): add SciCode dataset + 0-shot generation task

.gitignore

@@ -188,3 +188,10 @@ docs/handover/
 
 # Local-only model reference (not for OSS distribution)
 models.yaml
+
+# Local-only eval run configs (may contain live API keys)
+scicode-kimi.yaml
+scicode-kimi-slice.yaml
+scicode-kimi-p13.yaml
+scicode-kimi-bg-slice.yaml
+scicode-kimi-full-bg.yaml

pyproject.toml

@@ -51,6 +51,7 @@ math = [
   "math-verify[antlr4-11-0]==0.8.0",
   "regex>=2024.0.0",
 ]
+scicode = ["h5py>=3.11", "numpy<=2.2", "scipy>=1.16.3"]
 t-eval = ["numpy<=2.2", "sentence-transformers>=5.1.2"]
 
 [project.scripts]

sieval/community/scicode/__init__.py

@@ -0,0 +1,49 @@
+# Copyright 2024 The SciCode authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Vendored SciCode evaluation assets (github.com/scicode-bench/SciCode @ 69a8cfc).
+
+Local adaptation of the upstream prompt templates, code/HDF5 parsers, comparison
+helpers, and the three scientist-authored gold steps, plus a sandbox-program
+builder that inlines h5 targets so sieval's stateless code-eval service can run
+the tests.
+"""
+
+from sieval.community.scicode.harness import (
+    build_test_program,
+    encode_targets,
+)
+from sieval.community.scicode.parse import (
+    extract_function_name,
+    extract_python_script,
+    get_function_from_code,
+    process_hdf5_to_tuple,
+)
+from sieval.community.scicode.prompts import (
+    generate_prompt_with_steps,
+    is_special_step,
+    special_step_code,
+)
+
+__all__ = [
+    "build_test_program",
+    "encode_targets",
+    "extract_function_name",
+    "extract_python_script",
+    "get_function_from_code",
+    "process_hdf5_to_tuple",
+    "generate_prompt_with_steps",
+    "is_special_step",
+    "special_step_code",
+]

sieval/community/scicode/_cmp_upstream.py

@@ -0,0 +1,89 @@
+# Copyright 2024 The SciCode authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# adapted verbatim from https://github.com/scicode-bench/SciCode/blob/69a8cfc829fe8788a426ce8b5de6292366dce7ef/src/scicode/compare/cmp.py
+import numpy as np
+import scipy.sparse
+import sympy
+
+
+def are_dicts_close(dict1, dict2, atol=1e-8, rtol=1e-5):
+    dict1 = process_symbol_in_dict(dict1)
+    dict2 = process_symbol_in_dict(dict2)
+    # Check if both dictionaries have the same keys
+    if dict1.keys() != dict2.keys():
+        return False
+
+    # Check if the corresponding values are close
+    for key in dict1:
+        value1 = dict1[key]
+        value2 = dict2[key]
+        if isinstance(value1, (sympy.Symbol, str)):
+            if not value1 == value2:
+                return False
+        elif isinstance(value1, (scipy.sparse.csr_matrix, scipy.sparse.csc_matrix,
+                                 scipy.sparse.bsr_matrix, scipy.sparse.coo_matrix)):
+            value1 = value1.toarray()
+            value2 = value2.toarray()
+            if not np.allclose(value1, value2, atol=atol, rtol=rtol):
+                return False
+        # Use np.allclose to compare values
+        else:
+            try:
+                if not np.allclose(value1, value2, atol=atol, rtol=rtol):
+                    return False
+            except ValueError:
+                if not value1 == value2:
+                    return False
+
+    return True
+
+def process_symbol_in_dict(dict):
+    new_dict = {}
+    for key, value in dict.items():
+        new_dict[key] = value
+        if isinstance(value, sympy.Symbol):
+            new_dict[key] = str(value)
+        if isinstance(key, sympy.Symbol):
+            new_dict[str(key)] = dict[key]
+            new_dict.pop(key)
+    return new_dict
+
+def are_csc_matrix_close(matrix1, matrix2):
+    dense1 = matrix1.toarray()
+    dense2 = matrix2.toarray()
+    return np.allclose(dense1, dense2)
+
+def cmp_tuple_or_list(var1, var2):
+    if len(var1) != len(var2):
+        return False
+    for v1, v2 in zip(var1, var2):
+        if isinstance(v1, dict):
+            if not are_dicts_close(v1, v2):
+                return False
+        elif isinstance(v1, (scipy.sparse.csr_matrix, scipy.sparse.csc_matrix)):
+            if not are_csc_matrix_close(v1, v2):
+                return False
+        elif isinstance(v1, bool):
+            if not v1 == v2:
+                return False
+        else:
+            try:
+                if not np.allclose(v1, v2):
+                    return False
+            except ValueError as e:
+                print(e)
+                if not v1 == v2:
+                    return False
+    return True

sieval/community/scicode/harness.py

@@ -0,0 +1,83 @@
+"""Assemble a single self-contained test program for one SciCode sub-step.
+
+Upstream (eval/scripts/test_generated_code.py) runs each step inside an
+environment where ``scicode`` is installed and ``test_data.h5`` is on disk, then
+appends ``targets = process_hdf5_to_tuple(step_id, n)`` followed by each test
+case. sieval's code-eval sandbox is stateless, so instead we:
+
+  * read the numeric targets from the h5 on the eval side and inline them as a
+    pickled+zlib+base64 blob (mirrors how LiveCodeBench inlines its private
+    tests), and
+  * register an in-process ``scicode.compare.cmp`` module so the test bodies'
+    ``from scicode.compare.cmp import cmp_tuple_or_list`` imports resolve.
+
+The concatenated function code (dependencies + prior steps + current step) is
+supplied verbatim by the caller. The sandbox must provide the scientific stack
+the problems import (numpy / scipy / sympy / ...); h5py is NOT needed there.
+
+AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic)
+"""
+import base64
+import pickle
+import zlib
+from importlib import resources
+
+
+def _cmp_source() -> str:
+    return (
+        resources.files("sieval.community.scicode")
+        .joinpath("_cmp_upstream.py")
+        .read_text(encoding="utf-8")
+    )
+
+
+# Registers the vendored comparison helpers under the module path the upstream
+# test bodies import from, without requiring `scicode` to be installed.
+_SHIM_FOOTER = """
+import sys as _sys, types as _types
+
+_scicode_cmp = _types.ModuleType("scicode.compare.cmp")
+for _name in (
+    "cmp_tuple_or_list",
+    "are_dicts_close",
+    "are_csc_matrix_close",
+    "process_symbol_in_dict",
+):
+    setattr(_scicode_cmp, _name, globals()[_name])
+_scicode = _types.ModuleType("scicode")
+_scicode_compare = _types.ModuleType("scicode.compare")
+_scicode_compare.cmp = _scicode_cmp
+_scicode.compare = _scicode_compare
+_sys.modules.setdefault("scicode", _scicode)
+_sys.modules.setdefault("scicode.compare", _scicode_compare)
+_sys.modules["scicode.compare.cmp"] = _scicode_cmp
+"""
+
+
+def encode_targets(targets: list) -> str:
+    """Serialize h5 targets to a base64 string safe to embed in source."""
+    return base64.b64encode(zlib.compress(pickle.dumps(targets))).decode("ascii")
+
+
+def build_test_program(code_content: str, targets_b64: str, test_cases: list) -> str:
+    """Return a runnable program: solution code + shim + targets + test cases.
+
+    *code_content* is ``dependencies + prior-step funcs + current-step func``.
+    *targets_b64* is :func:`encode_targets` output. *test_cases* are the raw
+    upstream test-body strings; each references ``target`` (the i-th target).
+    """
+    parts = [
+        code_content,
+        "",
+        "# --- sieval scicode test harness (injected) ---",
+        "import base64 as _b64, zlib as _zlib, pickle as _pkl",
+        _cmp_source(),
+        _SHIM_FOOTER,
+        f'targets = _pkl.loads(_zlib.decompress(_b64.b64decode("{targets_b64}")))',
+        "",
+    ]
+    for idx, case in enumerate(test_cases):
+        parts.append(f"target = targets[{idx}]\n")
+        parts.append(case)
+        parts.append("")
+    return "\n".join(parts)