From d06ce16e462d0fbfb330f34d45fa26dd52af2887 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 15:45:35 -0700 Subject: [PATCH 01/10] feat(python-sdk): add LLMGeneratorProtocol for framework-agnostic model injection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces three types to sdks/python: - LLMGeneratorProtocol (typing.Protocol) — structural interface for injecting any LLM backend into evaluators without a hard framework dependency - LLMResponse (NamedTuple) — structured response aligned with OTel GenAI semconv (content, model, input_tokens, output_tokens) - GenerateConfig (dataclass) — temperature and max_tokens passthrough Refactors BaseEvaluator.execute_prompt_chain_step to accept an optional llm_provider: LLMGeneratorProtocol. When set, the protocol path formats the LangChain template to extract system/human strings, delegates the LLM call to the injected provider, and parses the JSON response via Pydantic directly. The existing LangChain path is unchanged and remains the default. Also improves _strip_json_fences to use JSONDecoder.raw_decode, correctly handling trailing prose and multiple JSON objects in LLM responses. Also adds *.egg-info/, dist/, build/, logs/ to root .gitignore. --- .gitignore | 9 ++ sdks/python/pyproject.toml | 2 +- .../learning_commons_evaluators/__init__.py | 8 + .../evaluators/base.py | 139 ++++++++++++++++++ .../schemas/__init__.py | 8 + .../schemas/llm_provider.py | 121 +++++++++++++++ .../python/tests/schemas/test_llm_provider.py | 106 +++++++++++++ 7 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py create mode 100644 sdks/python/tests/schemas/test_llm_provider.py diff --git a/.gitignore b/.gitignore index fb6507ee..1ee9c3bd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,15 @@ .venv/ __pycache__/ *.pyc +*.egg-info/ +dist/ +build/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Inspect AI eval logs +logs/ # Jupyter Notebook .ipynb_checkpoints/ diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index aea23087..b96dc345 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "learning-commons-evaluators" -version = "0.2.0" +version = "0.3.0" description = "Python SDK for Learning Commons educational evaluators" readme = "README.md" license = { text = "MIT" } diff --git a/sdks/python/src/learning_commons_evaluators/__init__.py b/sdks/python/src/learning_commons_evaluators/__init__.py index 6169df43..2ec1b989 100644 --- a/sdks/python/src/learning_commons_evaluators/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/__init__.py @@ -59,6 +59,11 @@ TextInputField, ) from learning_commons_evaluators.schemas.config import EvaluationSettings, LLMProvider +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.conventionality import ( ConventionalityEvaluationSettings, ConventionalityOutput, @@ -163,6 +168,9 @@ "create_config_telemetry_with_full_input", "create_logger", "create_silent_logger", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "get_logger", "wrap_provider_error", ] diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 9954c987..f41f31f7 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import json as _json +import re import time from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable @@ -19,6 +21,7 @@ from learning_commons_evaluators.schemas.config import ( EvaluationSettings, EvaluatorConfig, + LLMProvider, PromptSettings, ) from learning_commons_evaluators.schemas.errors import ( @@ -32,6 +35,11 @@ EvaluationInput, EvaluationResult, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -43,6 +51,31 @@ prompt_settings_to_extras_value, ) + +def _strip_json_fences(text: str) -> str: + """Strip markdown code fences and extract the first valid JSON object or array. + + Uses ``json.JSONDecoder.raw_decode`` to locate the first balanced JSON structure + and discard any surrounding prose or trailing text. This correctly handles: + - Markdown-fenced responses: ````json\\n{...}\\n``` `` + - Prose-prefixed responses: ``Here is the result:\\n{...}`` + - Trailing-prose responses: ``{...} Here is my reasoning...`` + """ + text = text.strip() + text = re.sub(r"^```(?:json)?\s*\n?", "", text) + text = re.sub(r"\n?```\s*$", "", text) + text = text.strip() + # Find the first { or [ and use raw_decode to extract the complete JSON structure, + # correctly discarding any trailing prose or a second JSON object in the response. + start = next((i for i, ch in enumerate(text) if ch in ("{", "[")), -1) + if start != -1: + try: + _, end = _json.JSONDecoder().raw_decode(text, start) + return text[start:end] + except _json.JSONDecodeError: + pass + return text + InputT = TypeVar("InputT", bound=EvaluationInput) OutputT = TypeVar("OutputT", bound=EvaluationResult) SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) @@ -74,9 +107,11 @@ def __init__( self, config: EvaluatorConfig, *, + llm_provider: LLMGeneratorProtocol | None = None, default_evaluation_settings: SettingsT | None = None, ) -> None: self.config = config + self._llm_provider = llm_provider if default_evaluation_settings is not None: self.default_evaluation_settings = default_evaluation_settings # TODO: validate config @@ -312,6 +347,14 @@ async def execute_prompt_chain_step( Parsed instance of ``parser_output_type`` when it is a model class; plain ``str`` when ``parser_output_type`` is omitted or ``None``. + Note: + **Execution path**: when ``self._llm_provider`` is set (injected at + construction via ``BaseEvaluator.__init__``), the *protocol path* is taken + — the LangChain template is formatted to extract system/human strings, the + injected provider is called directly, and JSON is parsed via Pydantic. + When ``self._llm_provider`` is ``None`` (default), the *LangChain path* + is taken and ``create_provider()`` is called internally. + Raises: ConfigurationError: No provider config for ``prompt_settings.provider_type``. OutputValidationError: The LLM response didn't satisfy the expected @@ -330,6 +373,102 @@ async def execute_prompt_chain_step( # Populated after a successful LLM invoke so we can attach usage even if parsing fails. token_usage: TokenUsage | None = None + if self._llm_provider is not None: + # ── Protocol path ────────────────────────────────────────────── + # Format the LangChain template to extract system/human strings, + # then delegate the actual LLM call to the injected provider. + # JSON parsing is handled directly via Pydantic — no LangChain parser needed. + provider = self._llm_provider + + async def _run_via_provider() -> BaseModel | str: + nonlocal token_usage + formatted = await template.aformat_messages(**chain_inputs) + system_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), "" + ) + human_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), "" + ) + try: + response: LLMResponse = await provider.generate( + system=system_str, + human=human_str, + config=GenerateConfig(temperature=prompt_settings.temperature), + ) + except EvaluatorError: + raise + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: + raise wrap_provider_error( + e, + provider=prompt_settings.provider_type, + model=prompt_settings.model, + ) from e + if response.input_tokens is not None or response.output_tokens is not None: + token_usage = TokenUsage( + provider_type=prompt_settings.provider_type, + model=response.model, + input_tokens=response.input_tokens or 0, + output_tokens=response.output_tokens or 0, + ) + if parser_output_type is None: + return response.content + raw_json = _strip_json_fences(response.content) + try: + if json_dict_normalizer is not None: + parsed_dict = _json.loads(raw_json) + if not isinstance(parsed_dict, dict): + raise OutputValidationError( + "Model output is not a JSON object", + provider=prompt_settings.provider_type, + model=response.model, + ) + try: + normalized = json_dict_normalizer(parsed_dict) + except (TypeError, ValueError) as norm_err: + raise OutputValidationError( + "Model output could not be normalized before validation", + provider=prompt_settings.provider_type, + model=response.model, + ) from norm_err + return parser_output_type.model_validate(normalized) + return parser_output_type.model_validate_json(raw_json) + except PydanticValidationError as e: + raise OutputValidationError( + provider=prompt_settings.provider_type, + model=response.model, + validation_errors=sanitize_pydantic_errors(e.errors()), + ) from e + except OutputValidationError: + raise + except Exception as e: + raise OutputValidationError( + provider=prompt_settings.provider_type, + model=response.model, + ) from e + + try: + return await self.execute_step( + step_name, + evaluation_metadata, + _run_via_provider, + extras={ + PROMPT_STEP_EXTRA_PROMPT_SETTINGS: prompt_settings_to_extras_value( + prompt_settings + ), + }, + ) + finally: + if token_usage is not None: + self.update_total_token_usage(token_usage, evaluation_metadata) + step = evaluation_metadata.step_details.get(step_name) + if step is not None: + step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE] = token_usage.model_dump( + mode="json" + ) + + # ── LangChain path (default, unchanged) ─────────────────────────── async def _run_chain() -> BaseModel | str: nonlocal token_usage try: diff --git a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py index 27caf004..bb41b809 100644 --- a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py @@ -33,6 +33,11 @@ InputSpec, TextInputSpec, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -81,5 +86,8 @@ "TextInputField", "TokenUsage", "InputValidationError", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "prompt_settings_to_extras_value", ] diff --git a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py new file mode 100644 index 00000000..7de3c594 --- /dev/null +++ b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py @@ -0,0 +1,121 @@ +"""LLM provider protocol and associated types for framework-agnostic model injection. + +These types define the interface that evaluation frameworks (Inspect AI, Braintrust, +Arize/Phoenix, Langfuse, etc.) implement to provide their own model execution to the SDK. + +``LLMGeneratorProtocol`` is a structural protocol (``typing.Protocol``) — integration +packages do not need to import or inherit from it. Any class with the correct ``generate`` +signature satisfies the protocol automatically for static type checkers. + +Response fields are aligned with OpenTelemetry GenAI semantic conventions: +https://opentelemetry.io/docs/specs/semconv/gen-ai/ +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import NamedTuple, Protocol + + +class LLMResponse(NamedTuple): + """Structured response from an LLM generation call. + + A ``NamedTuple`` — immutable, constructible by keyword or position, and + iterable for adapters that need to destructure the result. + + Fields are aligned with OpenTelemetry GenAI semantic conventions so that + observability adapters (Arize/Phoenix, Langfuse) can populate their spans + without additional parsing. + + Required fields (``content``, ``model``) must always be populated. + Optional fields should be populated whenever the underlying provider returns them. + """ + + content: str + """The model's text response.""" + + model: str + """The model that generated the response (``gen_ai.response.model``).""" + + input_tokens: int | None = None + """Number of input/prompt tokens consumed (``gen_ai.usage.input_tokens``).""" + + output_tokens: int | None = None + """Number of output/completion tokens generated (``gen_ai.usage.output_tokens``).""" + + +@dataclass +class GenerateConfig: + """Configuration for a single LLM generation call. + + All fields are optional. Adapters should apply whatever the underlying + provider supports and ignore the rest. + """ + + temperature: float | None = None + """Sampling temperature. 0.0 for deterministic output (recommended for evals).""" + + max_tokens: int | None = None + """Maximum number of tokens to generate.""" + + +class LLMGeneratorProtocol(Protocol): + """Structural protocol for LLM generation adapters. + + Implement this protocol in an integration package to allow the SDK to call + your framework's model system. + + No import of this class is required in the implementing package. Structural + conformance (correct ``generate`` signature) is sufficient for static type + checkers. + + **Lifecycle**: if your adapter holds a connection pool or HTTP session, add + ``async def aclose(self) -> None`` and call it when done. The protocol does + not include ``aclose`` so that stateless adapters remain fully conformant + without boilerplate. Callers that want to support teardown should use + ``hasattr(adapter, "aclose")``. + + Example:: + + from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMResponse, + ) + + class MyFrameworkAdapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + response = await my_framework.call(system, human) + return LLMResponse( + content=response.text, + model=response.model_name, + input_tokens=response.usage.input, + output_tokens=response.usage.output, + ) + """ + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + """Generate a response from the LLM. + + Args: + system: The system prompt. + human: The human/user prompt. + config: Optional generation configuration. Adapters apply whatever + fields the underlying provider supports and ignore the rest. + + Returns: + ``LLMResponse`` with at minimum ``content`` and ``model`` populated. + Populate optional fields (token counts) whenever the provider returns them. + """ + ... diff --git a/sdks/python/tests/schemas/test_llm_provider.py b/sdks/python/tests/schemas/test_llm_provider.py new file mode 100644 index 00000000..e819df3f --- /dev/null +++ b/sdks/python/tests/schemas/test_llm_provider.py @@ -0,0 +1,106 @@ +"""Tests for LLMGeneratorProtocol, LLMResponse, and GenerateConfig.""" + +from __future__ import annotations + +import learning_commons_evaluators +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class TestLLMResponse: + def test_required_fields(self): + r = LLMResponse(content="hello", model="anthropic/claude-opus-4-8") + assert r.content == "hello" + assert r.model == "anthropic/claude-opus-4-8" + assert r.input_tokens is None + assert r.output_tokens is None + + def test_optional_token_fields(self): + r = LLMResponse(content="text", model="test", input_tokens=100, output_tokens=50) + assert r.input_tokens == 100 + assert r.output_tokens == 50 + + + +class TestGenerateConfig: + def test_defaults(self): + c = GenerateConfig() + assert c.temperature is None + assert c.max_tokens is None + + def test_with_values(self): + c = GenerateConfig(temperature=0.0, max_tokens=512) + assert c.temperature == 0.0 + assert c.max_tokens == 512 + + +class TestLLMGeneratorProtocol: + async def test_generate_returns_llm_response(self): + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + return LLMResponse(content="hi", model="m", input_tokens=5, output_tokens=2) + + result = await Adapter().generate(system="sys", human="hello") + assert result.content == "hi" + assert result.model == "m" + assert result.input_tokens == 5 + + async def test_generate_config_passed_through(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + cfg = GenerateConfig(temperature=0.0, max_tokens=256) + await Adapter().generate(system="sys", human="hello", config=cfg) + assert received[0] is cfg + assert received[0].temperature == 0.0 + + async def test_generate_config_none_by_default(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + await Adapter().generate(system="sys", human="hello") + assert received[0] is None + + def test_structural_conformance(self): + """Static type checkers validate this assignment — no subclassing required. + + LLMGeneratorProtocol is not @runtime_checkable; isinstance() is not available. + Conformance is enforced at type-check time (mypy/pyright) by the annotation below. + """ + + class Adapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + return LLMResponse(content="", model="test") + + # mypy/pyright validate this structurally + _adapter: LLMGeneratorProtocol = Adapter() + assert _adapter is not None # runtime no-op; static check is the value + + +def test_exported_from_package(): + assert "GenerateConfig" in learning_commons_evaluators.__all__ + assert "LLMGeneratorProtocol" in learning_commons_evaluators.__all__ + assert "LLMResponse" in learning_commons_evaluators.__all__ From 5828620b8044849f5a63410a5f946bd4fb628791 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 21:12:03 -0700 Subject: [PATCH 02/10] fix(python-sdk): remove unused LLMProvider import from base.py --- sdks/python/src/learning_commons_evaluators/evaluators/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index f41f31f7..b96d31ef 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -21,7 +21,6 @@ from learning_commons_evaluators.schemas.config import ( EvaluationSettings, EvaluatorConfig, - LLMProvider, PromptSettings, ) from learning_commons_evaluators.schemas.errors import ( From 3ecaf88edf87b6c49fd2f2767e43146d804699b3 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 21:23:40 -0700 Subject: [PATCH 03/10] style(python-sdk): apply ruff formatter to base.py and test_llm_provider.py --- sdks/python/src/learning_commons_evaluators/evaluators/base.py | 1 + sdks/python/tests/schemas/test_llm_provider.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index b96d31ef..080c5be0 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -75,6 +75,7 @@ def _strip_json_fences(text: str) -> str: pass return text + InputT = TypeVar("InputT", bound=EvaluationInput) OutputT = TypeVar("OutputT", bound=EvaluationResult) SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) diff --git a/sdks/python/tests/schemas/test_llm_provider.py b/sdks/python/tests/schemas/test_llm_provider.py index e819df3f..b0782097 100644 --- a/sdks/python/tests/schemas/test_llm_provider.py +++ b/sdks/python/tests/schemas/test_llm_provider.py @@ -24,7 +24,6 @@ def test_optional_token_fields(self): assert r.output_tokens == 50 - class TestGenerateConfig: def test_defaults(self): c = GenerateConfig() From ca64de28e8be836aac45f18e4a9719cbb383b11a Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 21:32:30 -0700 Subject: [PATCH 04/10] test(python-sdk): add protocol path coverage for execute_prompt_chain_step Adds TestExecutePromptChainStepProtocolPath (13 tests) covering the llm_provider injection path in BaseEvaluator.execute_prompt_chain_step: - Raw string return when parser_output_type=None - Clean JSON parse - Markdown fence stripping - Trailing prose stripping (JSON followed by explanation text) - Leading prose stripping (prose before JSON) - json_dict_normalizer path - Non-dict JSON raises OutputValidationError on normalizer path - Malformed JSON raises OutputValidationError - Schema mismatch raises OutputValidationError - Token usage recorded in step extras and total_token_usage - Token usage absent when LLMResponse has None tokens - Provider RuntimeError wrapped as APIError - EvaluatorError from provider re-raised unchanged - KeyboardInterrupt from provider propagated --- sdks/python/tests/evaluators/test_base.py | 241 ++++++++++++++++++++++ 1 file changed, 241 insertions(+) diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index f595af9c..998c6808 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -857,3 +857,244 @@ def passthrough(d: dict) -> dict: assert "JSON object" in str(exc_info.value) assert exc_info.value.provider is LLMProvider.GOOGLE assert exc_info.value.model == "gemini-2.0-flash" + + +# --------------------------------------------------------------------------- +# execute_prompt_chain_step — protocol path (llm_provider injected) +# --------------------------------------------------------------------------- + + +from learning_commons_evaluators.schemas.llm_provider import LLMResponse # noqa: E402 + + +def _make_adapter( + content: str, + model: str = "test-model", + input_tokens: int | None = 10, + output_tokens: int | None = 5, +) -> AsyncMock: + """Minimal mock that satisfies LLMGeneratorProtocol.generate().""" + adapter = AsyncMock() + adapter.generate = AsyncMock( + return_value=LLMResponse( + content=content, model=model, input_tokens=input_tokens, output_tokens=output_tokens + ) + ) + return adapter + + +_PROTO_SETTINGS = PromptSettings( + provider_type=LLMProvider.ANTHROPIC, + model="claude-opus-4-8", + temperature=0.0, +) + +_PROTO_TEMPLATE = ChatPromptTemplate.from_messages( + [("system", "You are a grader."), ("human", "{input}")] +) + + +class TestExecutePromptChainStepProtocolPath: + """Protocol path: llm_provider injected — LangChain provider is never called.""" + + def _ev(self, adapter: AsyncMock) -> _StubEvaluator: + return _StubEvaluator(create_config_no_telemetry(), llm_provider=adapter) + + async def test_returns_raw_string_when_parser_type_is_none(self, evaluation_metadata): + ev = self._ev(_make_adapter("plain prose")) + out = await ev.execute_prompt_chain_step( + step_name="raw", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=None, + ) + assert out == "plain prose" + + async def test_parses_clean_json(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(result, _ChainOutput) + assert result.label == "ok" + assert result.score == 7 + + async def test_strips_markdown_fences(self, evaluation_metadata): + fenced = f"```json\n{_CHAIN_JSON}\n```" + ev = self._ev(_make_adapter(fenced)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_trailing_prose(self, evaluation_metadata): + with_prose = f"{_CHAIN_JSON}\n\nHere is my reasoning for this score." + ev = self._ev(_make_adapter(with_prose)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_leading_prose(self, evaluation_metadata): + with_prefix = f"Here is the result:\n{_CHAIN_JSON}" + ev = self._ev(_make_adapter(with_prefix)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_json_dict_normalizer_path(self, evaluation_metadata): + class _Out(BaseModel): + n: int + doubled: int + + ev = self._ev(_make_adapter('{"n": 3}')) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: {**d, "doubled": d["n"] * 2}, + ) + assert result.n == 3 + assert result.doubled == 6 + + async def test_non_dict_json_in_normalizer_path_raises_output_validation_error( + self, evaluation_metadata + ): + class _Out(BaseModel): + n: int + + ev = self._ev(_make_adapter('["not", "an", "object"]')) + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: d, + ) + assert "JSON object" in str(exc_info.value) + + async def test_malformed_json_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter("not json at all")) + with pytest.raises(OutputValidationError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_schema_mismatch_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter('{"label": "only"}')) # missing required `score` + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, PydanticValidationError) + + async def test_token_usage_recorded_in_step_extras_and_total(self, evaluation_metadata): + ev = self._ev( + _make_adapter(_CHAIN_JSON, model="claude-opus-4-8", input_tokens=42, output_tokens=17) + ) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + step = evaluation_metadata.step_details["main"] + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["input_tokens"] == 42 + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["output_tokens"] == 17 + assert evaluation_metadata.total_token_usage[LLMProvider.ANTHROPIC].input_tokens == 42 + + async def test_token_usage_absent_when_llm_response_has_none_tokens(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON, input_tokens=None, output_tokens=None)) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert not evaluation_metadata.total_token_usage + + async def test_provider_error_wrapped_as_api_error(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=RuntimeError("network timeout")) + ev = self._ev(adapter) + with pytest.raises(APIError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, RuntimeError) + + async def test_evaluator_error_from_provider_reraises_unchanged(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=EvaluatorError("already wrapped")) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError, match="already wrapped"): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_keyboard_interrupt_from_provider_propagates(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=KeyboardInterrupt) + ev = self._ev(adapter) + with pytest.raises(KeyboardInterrupt): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) From 7418a24e026606a50705b40bd4ab96f0a36f173e Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 22:00:18 -0700 Subject: [PATCH 05/10] fix(python-sdk): address independent code review findings - Revert version bump to 0.2.0 (release-please handles this on merge) - Add model field to GenerateConfig so adapters can see which model the evaluator expects without reaching into prompt_settings - Move template.aformat_messages() inside try block so missing template variables raise EvaluatorError rather than bare KeyError - Add ValueError when human_str is empty; DEBUG log when system_str is empty - Extract _parse_json_output() helper to deduplicate JSON parsing and error wrapping between the protocol and LangChain paths - Add tests: assert adapter.generate() called with correct system/human, human-only template passes empty system string, missing variable raises EvaluatorError --- sdks/python/pyproject.toml | 2 +- .../evaluators/base.py | 163 ++++++++++-------- .../schemas/llm_provider.py | 12 ++ sdks/python/tests/evaluators/test_base.py | 51 ++++++ 4 files changed, 156 insertions(+), 72 deletions(-) diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index b96dc345..aea23087 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "learning-commons-evaluators" -version = "0.3.0" +version = "0.2.0" description = "Python SDK for Learning Commons educational evaluators" readme = "README.md" license = { text = "MIT" } diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 080c5be0..cd81d85c 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -50,6 +50,57 @@ prompt_settings_to_extras_value, ) +# TypeVars used by module-level helpers and the BaseEvaluator generic class. +InputT = TypeVar("InputT", bound=EvaluationInput) +OutputT = TypeVar("OutputT", bound=EvaluationResult) +SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) +StepResultT = TypeVar("StepResultT") +ParsedT = TypeVar("ParsedT", bound=BaseModel) + + +def _parse_json_output( + raw: str, + parser_output_type: type[ParsedT], + json_dict_normalizer: Callable[[dict], dict] | None, + provider_type: Any, + model: str, +) -> ParsedT: + """Parse a raw JSON string into ``parser_output_type``, wrapping errors consistently. + + Shared by both the protocol path and (for the normalizer branch) the LangChain path + so that error handling is symmetric and normaliser logic lives in one place. + """ + raw = _strip_json_fences(raw) + try: + if json_dict_normalizer is not None: + parsed_dict = _json.loads(raw) + if not isinstance(parsed_dict, dict): + raise OutputValidationError( + "Model output is not a JSON object", + provider=provider_type, + model=model, + ) + try: + normalized = json_dict_normalizer(parsed_dict) + except (TypeError, ValueError) as norm_err: + raise OutputValidationError( + "Model output could not be normalized before validation", + provider=provider_type, + model=model, + ) from norm_err + return parser_output_type.model_validate(normalized) + return parser_output_type.model_validate_json(raw) + except PydanticValidationError as e: + raise OutputValidationError( + provider=provider_type, + model=model, + validation_errors=sanitize_pydantic_errors(e.errors()), + ) from e + except OutputValidationError: + raise + except Exception as e: + raise OutputValidationError(provider=provider_type, model=model) from e + def _strip_json_fences(text: str) -> str: """Strip markdown code fences and extract the first valid JSON object or array. @@ -76,13 +127,6 @@ def _strip_json_fences(text: str) -> str: return text -InputT = TypeVar("InputT", bound=EvaluationInput) -OutputT = TypeVar("OutputT", bound=EvaluationResult) -SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) -StepResultT = TypeVar("StepResultT") -ParsedT = TypeVar("ParsedT", bound=BaseModel) - - class BaseEvaluator(ABC, Generic[InputT, OutputT, SettingsT]): """ Abstract base class for all evaluators. @@ -382,18 +426,36 @@ async def execute_prompt_chain_step( async def _run_via_provider() -> BaseModel | str: nonlocal token_usage - formatted = await template.aformat_messages(**chain_inputs) - system_str = next( - (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), "" - ) - human_str = next( - (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), "" - ) try: + # Template formatting is inside try so missing variables become EvaluatorErrors, + # not bare KeyErrors — matching the error contract of the LangChain path. + formatted = await template.aformat_messages(**chain_inputs) + system_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), + "", + ) + human_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), + "", + ) + if not human_str: + raise ValueError( + f"Template for step '{step_name}' produced no human message. " + 'Ensure the template contains at least one ("human", ...) turn.' + ) + if not system_str: + self.config.logger.debug( + "No system message in template for step '%s'; " + "passing empty string to adapter.", + step_name, + ) response: LLMResponse = await provider.generate( system=system_str, human=human_str, - config=GenerateConfig(temperature=prompt_settings.temperature), + config=GenerateConfig( + temperature=prompt_settings.temperature, + model=prompt_settings.model, + ), ) except EvaluatorError: raise @@ -414,39 +476,13 @@ async def _run_via_provider() -> BaseModel | str: ) if parser_output_type is None: return response.content - raw_json = _strip_json_fences(response.content) - try: - if json_dict_normalizer is not None: - parsed_dict = _json.loads(raw_json) - if not isinstance(parsed_dict, dict): - raise OutputValidationError( - "Model output is not a JSON object", - provider=prompt_settings.provider_type, - model=response.model, - ) - try: - normalized = json_dict_normalizer(parsed_dict) - except (TypeError, ValueError) as norm_err: - raise OutputValidationError( - "Model output could not be normalized before validation", - provider=prompt_settings.provider_type, - model=response.model, - ) from norm_err - return parser_output_type.model_validate(normalized) - return parser_output_type.model_validate_json(raw_json) - except PydanticValidationError as e: - raise OutputValidationError( - provider=prompt_settings.provider_type, - model=response.model, - validation_errors=sanitize_pydantic_errors(e.errors()), - ) from e - except OutputValidationError: - raise - except Exception as e: - raise OutputValidationError( - provider=prompt_settings.provider_type, - model=response.model, - ) from e + return _parse_json_output( + response.content, + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + response.model, + ) try: return await self.execute_step( @@ -481,29 +517,14 @@ async def _run_chain() -> BaseModel | str: from langchain_core.output_parsers.json import JsonOutputParser if json_dict_normalizer is not None: - loose = JsonOutputParser() - parsed_dict = await loose.ainvoke(ai_message) - if not isinstance(parsed_dict, dict): - # JSON parsed cleanly but the top-level value isn't an object - # (e.g. the LLM returned a JSON array or scalar). That's an - # output-shape failure, not a parse failure — surface it as - # OutputValidationError so callers can treat it consistently - # with schema-mismatch errors, and avoid the TypeError that - # ``dict(parsed_dict)`` would raise on a non-dict. - raise OutputValidationError( - "Model output is not a JSON object", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) - try: - normalized = json_dict_normalizer(parsed_dict) - except (TypeError, ValueError) as norm_err: - raise OutputValidationError( - "Model output could not be normalized before validation", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) from norm_err - return parser_output_type.model_validate(normalized) + # Use the shared helper so normalizer + error-wrapping logic stays in one place. + return _parse_json_output( + str(ai_message.content), + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + prompt_settings.model, + ) parser = JsonOutputParser(pydantic_object=parser_output_type) raw = await parser.ainvoke(ai_message) diff --git a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py index 7de3c594..0ccf489b 100644 --- a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py +++ b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py @@ -58,6 +58,18 @@ class GenerateConfig: max_tokens: int | None = None """Maximum number of tokens to generate.""" + model: str | None = None + """Model identifier to request from the provider (e.g. ``"claude-opus-4-8"``). + + Adapters should use this when set and ignore it otherwise — the contract is + identical to all other ``GenerateConfig`` fields. When ``None`` (default), + the adapter uses whatever model it was constructed with. + + Populated from ``PromptSettings.model`` on the protocol path so that + adapter authors can inspect which model the evaluator expects without + reaching into ``prompt_settings`` directly. + """ + class LLMGeneratorProtocol(Protocol): """Structural protocol for LLM generation adapters. diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index 998c6808..863d36e0 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -1098,3 +1098,54 @@ async def test_keyboard_interrupt_from_provider_propagates(self, evaluation_meta chain_inputs={"input": "Hello"}, parser_output_type=_ChainOutput, ) + + async def test_adapter_called_with_formatted_system_and_human(self, evaluation_metadata): + """Template formatting actually reaches the adapter with the correct strings.""" + from unittest.mock import ANY + + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with( + system="You are a grader.", + human="Hello", + config=ANY, + ) + + async def test_human_only_template_passes_empty_system(self, evaluation_metadata): + """Templates with no system turn pass empty string to the adapter without error.""" + from unittest.mock import ANY + + human_only = ChatPromptTemplate.from_messages([("human", "{input}")]) + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=human_only, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with(system="", human="Hello", config=ANY) + + async def test_template_with_missing_variable_raises_evaluator_error(self, evaluation_metadata): + """A missing template variable becomes an EvaluatorError, not a bare KeyError.""" + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={}, # missing required "input" variable + parser_output_type=_ChainOutput, + ) From 7d75e0d5c0f7088e15b069613c243e039645fa30 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 22:12:41 -0700 Subject: [PATCH 06/10] style: remove redundant inline comments in base.py --- .../learning_commons_evaluators/evaluators/base.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index cd81d85c..70d2b412 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -50,7 +50,6 @@ prompt_settings_to_extras_value, ) -# TypeVars used by module-level helpers and the BaseEvaluator generic class. InputT = TypeVar("InputT", bound=EvaluationInput) OutputT = TypeVar("OutputT", bound=EvaluationResult) SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) @@ -418,17 +417,14 @@ async def execute_prompt_chain_step( token_usage: TokenUsage | None = None if self._llm_provider is not None: - # ── Protocol path ────────────────────────────────────────────── - # Format the LangChain template to extract system/human strings, - # then delegate the actual LLM call to the injected provider. - # JSON parsing is handled directly via Pydantic — no LangChain parser needed. + # ── Protocol path ───────────────────────────────────────────── provider = self._llm_provider async def _run_via_provider() -> BaseModel | str: nonlocal token_usage try: - # Template formatting is inside try so missing variables become EvaluatorErrors, - # not bare KeyErrors — matching the error contract of the LangChain path. + # Inside try: missing template variables become EvaluatorErrors, + # not bare KeyErrors — consistent with the LangChain path's error contract. formatted = await template.aformat_messages(**chain_inputs) system_str = next( (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), @@ -517,7 +513,6 @@ async def _run_chain() -> BaseModel | str: from langchain_core.output_parsers.json import JsonOutputParser if json_dict_normalizer is not None: - # Use the shared helper so normalizer + error-wrapping logic stays in one place. return _parse_json_output( str(ai_message.content), parser_output_type, From 2e1df68e1454a01a1427ece3a7ac829b6ac169ae Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 15:46:08 -0700 Subject: [PATCH 07/10] feat: add Inspect AI, Arize, Langfuse, and Braintrust integration packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds four packages under integrations/ that each implement LLMGeneratorProtocol (introduced in the SDK PR) for their respective platform: integrations/inspect-python → learning-commons-inspect-scorers - InspectModelAdapter wraps Inspect's get_model() so the GLA scorer uses Inspect's model system rather than LangChain directly - gla_scorer() Inspect scorer for grade-level appropriateness evaluation integrations/arize-python → learning-commons-arize-scorers - PhoenixTracingAdapter: OTel decorator that emits OpenInference llm.* and gen_ai.* spans; capture_message_content=False by default (K-12 privacy) integrations/langfuse-python → learning-commons-langfuse-scorers - LangfuseTracingAdapter: decorator that records Langfuse v2 generations; pinned to langfuse<3.0.0 pending migration to OTel-based v3 API integrations/braintrust-python → learning-commons-braintrust-scorers - BraintrustAnthropicAdapter: uses auto_instrument() for transparent tracing - BraintrustProxyAdapter: routes calls through Braintrust AI Proxy with no Braintrust SDK dependency All adapters implement LLMGeneratorProtocol structurally (typing.Protocol) and are composable as decorators: PhoenixTracingAdapter(LangfuseTracingAdapter(InspectModelAdapter("..."))) Also updates release-please-config.json and manifest to track all four packages. --- .release-please-manifest.json | 6 +- integrations/arize-python/.gitignore | 6 + integrations/arize-python/CHANGELOG.md | 1 + integrations/arize-python/pyproject.toml | 68 ++++ .../__init__.py | 5 + .../learning_commons_arize_scorers/adapter.py | 79 ++++ .../learning_commons_arize_scorers/py.typed | 0 .../arize-python/tests/test_adapter.py | 177 +++++++++ integrations/braintrust-python/.gitignore | 6 + integrations/braintrust-python/CHANGELOG.md | 7 + integrations/braintrust-python/pyproject.toml | 70 ++++ .../__init__.py | 28 ++ .../adapter.py | 155 ++++++++ .../py.typed | 0 .../braintrust-python/tests/test_adapter.py | 296 +++++++++++++++ integrations/inspect-python/.gitignore | 6 + integrations/inspect-python/CHANGELOG.md | 1 + integrations/inspect-python/README.md | 79 ++++ integrations/inspect-python/pyproject.toml | 74 ++++ .../__init__.py | 6 + .../_registry.py | 8 + .../adapter.py | 70 ++++ .../learning_commons_inspect_scorers/gla.py | 151 ++++++++ .../learning_commons_inspect_scorers/py.typed | 0 .../inspect-python/tests/test_gla_scorer.py | 341 ++++++++++++++++++ integrations/langfuse-python/.gitignore | 6 + integrations/langfuse-python/CHANGELOG.md | 1 + integrations/langfuse-python/pyproject.toml | 70 ++++ .../__init__.py | 5 + .../adapter.py | 96 +++++ .../py.typed | 0 .../langfuse-python/tests/test_adapter.py | 181 ++++++++++ release-please-config.json | 32 ++ 33 files changed, 2030 insertions(+), 1 deletion(-) create mode 100644 integrations/arize-python/.gitignore create mode 100644 integrations/arize-python/CHANGELOG.md create mode 100644 integrations/arize-python/pyproject.toml create mode 100644 integrations/arize-python/src/learning_commons_arize_scorers/__init__.py create mode 100644 integrations/arize-python/src/learning_commons_arize_scorers/adapter.py create mode 100644 integrations/arize-python/src/learning_commons_arize_scorers/py.typed create mode 100644 integrations/arize-python/tests/test_adapter.py create mode 100644 integrations/braintrust-python/.gitignore create mode 100644 integrations/braintrust-python/CHANGELOG.md create mode 100644 integrations/braintrust-python/pyproject.toml create mode 100644 integrations/braintrust-python/src/learning_commons_braintrust_scorers/__init__.py create mode 100644 integrations/braintrust-python/src/learning_commons_braintrust_scorers/adapter.py create mode 100644 integrations/braintrust-python/src/learning_commons_braintrust_scorers/py.typed create mode 100644 integrations/braintrust-python/tests/test_adapter.py create mode 100644 integrations/inspect-python/.gitignore create mode 100644 integrations/inspect-python/CHANGELOG.md create mode 100644 integrations/inspect-python/README.md create mode 100644 integrations/inspect-python/pyproject.toml create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed create mode 100644 integrations/inspect-python/tests/test_gla_scorer.py create mode 100644 integrations/langfuse-python/.gitignore create mode 100644 integrations/langfuse-python/CHANGELOG.md create mode 100644 integrations/langfuse-python/pyproject.toml create mode 100644 integrations/langfuse-python/src/learning_commons_langfuse_scorers/__init__.py create mode 100644 integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py create mode 100644 integrations/langfuse-python/src/learning_commons_langfuse_scorers/py.typed create mode 100644 integrations/langfuse-python/tests/test_adapter.py diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 297471d8..8f8d7d06 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,5 +1,9 @@ { "evals/prompts": "1.5.0", "sdks/python": "0.2.0", - "sdks/typescript": "0.7.0" + "sdks/typescript": "0.7.0", + "integrations/inspect-python": "0.1.0", + "integrations/langfuse-python": "0.1.0", + "integrations/arize-python": "0.1.0", + "integrations/braintrust-python": "0.1.0" } diff --git a/integrations/arize-python/.gitignore b/integrations/arize-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/arize-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/arize-python/CHANGELOG.md b/integrations/arize-python/CHANGELOG.md new file mode 100644 index 00000000..825c32f0 --- /dev/null +++ b/integrations/arize-python/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/integrations/arize-python/pyproject.toml b/integrations/arize-python/pyproject.toml new file mode 100644 index 00000000..163a67e4 --- /dev/null +++ b/integrations/arize-python/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-arize-scorers" +version = "0.1.0" +description = "Arize/Phoenix OTel tracing adapter for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "arize", "phoenix", "opentelemetry", "tracing"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + "opentelemetry-api>=1.0.0", +] + +[project.optional-dependencies] +dev = [ + "opentelemetry-sdk>=1.0.0", + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/arize-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_arize_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/arize-python/src/learning_commons_arize_scorers/__init__.py b/integrations/arize-python/src/learning_commons_arize_scorers/__init__.py new file mode 100644 index 00000000..7b0c3a23 --- /dev/null +++ b/integrations/arize-python/src/learning_commons_arize_scorers/__init__.py @@ -0,0 +1,5 @@ +"""Learning Commons Arize scorers — OpenInference OTel tracing adapter for LC evaluators.""" + +from learning_commons_arize_scorers.adapter import PhoenixTracingAdapter + +__all__ = ["PhoenixTracingAdapter"] diff --git a/integrations/arize-python/src/learning_commons_arize_scorers/adapter.py b/integrations/arize-python/src/learning_commons_arize_scorers/adapter.py new file mode 100644 index 00000000..b6c325d4 --- /dev/null +++ b/integrations/arize-python/src/learning_commons_arize_scorers/adapter.py @@ -0,0 +1,79 @@ +"""PhoenixTracingAdapter — decorates any LLMGeneratorProtocol with OpenInference OTel spans.""" + +from __future__ import annotations + +from opentelemetry import trace +from opentelemetry.trace import Tracer +from opentelemetry.trace.status import Status, StatusCode + +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class PhoenixTracingAdapter: + """Decorator adapter: wraps any LLMGeneratorProtocol, emits OpenInference OTel spans. + + Composes with any other adapter:: + + from learning_commons_arize_scorers import PhoenixTracingAdapter + from learning_commons_inspect_scorers.adapter import InspectModelAdapter + + adapter = PhoenixTracingAdapter(InspectModelAdapter("anthropic/claude-opus-4-8")) + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) + + Args: + inner: The underlying adapter to delegate generation to. + tracer: OTel Tracer instance. Defaults to a tracer named + ``"learning_commons_arize_scorers"``. + capture_message_content: If ``True``, writes system and human prompt text + and the model response into span attributes. Defaults to ``False``. + + .. warning:: + Enabling this may capture student-submitted text and other PII + into your observability backend. Ensure your data handling + controls (FERPA, COPPA for K-12) permit this before enabling. + """ + + def __init__( + self, + inner: LLMGeneratorProtocol, + tracer: Tracer | None = None, + *, + capture_message_content: bool = False, + ) -> None: + self._inner = inner + self._tracer = tracer or trace.get_tracer("learning_commons_arize_scorers") + self._capture_message_content = capture_message_content + + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + with self._tracer.start_as_current_span("llm.generate") as span: + span.set_attribute("openinference.span.kind", "LLM") + span.set_attribute("gen_ai.operation.name", "chat") + if self._capture_message_content: + span.set_attribute("llm.input_messages.0.message.role", "system") + span.set_attribute("llm.input_messages.0.message.content", system) + span.set_attribute("llm.input_messages.1.message.role", "user") + span.set_attribute("llm.input_messages.1.message.content", human) + try: + response = await self._inner.generate(system=system, human=human, config=config) + span.set_attribute("gen_ai.response.model", response.model) + span.set_attribute("llm.model_name", response.model) + if response.input_tokens is not None: + span.set_attribute("gen_ai.usage.input_tokens", response.input_tokens) + span.set_attribute("llm.token_count.prompt", response.input_tokens) + if response.output_tokens is not None: + span.set_attribute("gen_ai.usage.output_tokens", response.output_tokens) + span.set_attribute("llm.token_count.completion", response.output_tokens) + if self._capture_message_content: + span.set_attribute("llm.output_messages.0.message.role", "assistant") + span.set_attribute("llm.output_messages.0.message.content", response.content) + return response + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR)) + raise diff --git a/integrations/arize-python/src/learning_commons_arize_scorers/py.typed b/integrations/arize-python/src/learning_commons_arize_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/arize-python/tests/test_adapter.py b/integrations/arize-python/tests/test_adapter.py new file mode 100644 index 00000000..d9ddb279 --- /dev/null +++ b/integrations/arize-python/tests/test_adapter.py @@ -0,0 +1,177 @@ +"""Tests for PhoenixTracingAdapter using InMemorySpanExporter.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse +from learning_commons_arize_scorers import PhoenixTracingAdapter + + +# ── Fixtures ────────────────────────────────────────────────────────────────── + + +@pytest.fixture() +def exporter() -> InMemorySpanExporter: + return InMemorySpanExporter() + + +@pytest.fixture() +def tracer(exporter: InMemorySpanExporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider.get_tracer("test") + + +def _make_response( + content: str = "The answer is 42.", + model: str = "claude-test", + input_tokens: int | None = 10, + output_tokens: int | None = 5, +) -> LLMResponse: + return LLMResponse( + content=content, + model=model, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + +def _make_inner(response: LLMResponse | None = None, side_effect=None) -> AsyncMock: + mock = AsyncMock() + mock.generate = AsyncMock( + return_value=response or _make_response(), + side_effect=side_effect, + ) + return mock + + +# ── Basic span emission ─────────────────────────────────────────────────────── + + +class TestPhoenixTracingAdapterSpans: + async def test_emits_one_span_per_call(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="You are helpful.", human="What is 6×7?") + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "llm.generate" + + async def test_span_kind_attribute(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + span = exporter.get_finished_spans()[0] + assert span.attributes["openinference.span.kind"] == "LLM" + assert span.attributes["gen_ai.operation.name"] == "chat" + + async def test_input_message_attributes(self, tracer, exporter): + inner = _make_inner() + # capture_message_content=True required — off by default for K-12 privacy compliance + adapter = PhoenixTracingAdapter(inner, tracer=tracer, capture_message_content=True) + await adapter.generate(system="Be concise.", human="Hello?") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["llm.input_messages.0.message.role"] == "system" + assert attrs["llm.input_messages.0.message.content"] == "Be concise." + assert attrs["llm.input_messages.1.message.role"] == "user" + assert attrs["llm.input_messages.1.message.content"] == "Hello?" + + async def test_input_message_attributes_absent_by_default(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) # capture_message_content=False + await adapter.generate(system="Be concise.", human="Hello?") + attrs = exporter.get_finished_spans()[0].attributes + assert "llm.input_messages.0.message.content" not in attrs + assert "llm.input_messages.1.message.content" not in attrs + + async def test_output_message_attributes(self, tracer, exporter): + inner = _make_inner(_make_response(content="Hi there!")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer, capture_message_content=True) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["llm.output_messages.0.message.role"] == "assistant" + assert attrs["llm.output_messages.0.message.content"] == "Hi there!" + + async def test_model_attributes(self, tracer, exporter): + inner = _make_inner(_make_response(model="claude-opus-4")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["gen_ai.response.model"] == "claude-opus-4" + assert attrs["llm.model_name"] == "claude-opus-4" + + async def test_token_count_attributes(self, tracer, exporter): + inner = _make_inner(_make_response(input_tokens=20, output_tokens=8)) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["gen_ai.usage.input_tokens"] == 20 + assert attrs["llm.token_count.prompt"] == 20 + assert attrs["gen_ai.usage.output_tokens"] == 8 + assert attrs["llm.token_count.completion"] == 8 + + async def test_none_token_counts_omitted(self, tracer, exporter): + inner = _make_inner(_make_response(input_tokens=None, output_tokens=None)) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert "gen_ai.usage.input_tokens" not in attrs + assert "gen_ai.usage.output_tokens" not in attrs + + async def test_passes_config_to_inner(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + cfg = GenerateConfig(temperature=0.3, max_tokens=512) + await adapter.generate(system="sys", human="user", config=cfg) + inner.generate.assert_called_once_with(system="sys", human="user", config=cfg) + + async def test_returns_inner_response(self, tracer, exporter): + response = _make_response(content="42", model="gpt-test", input_tokens=3, output_tokens=1) + inner = _make_inner(response) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + result = await adapter.generate(system="sys", human="user") + assert result is response + + +# ── Exception handling ──────────────────────────────────────────────────────── + + +class TestPhoenixTracingAdapterErrors: + async def test_exception_is_recorded_on_span(self, tracer, exporter): + inner = _make_inner(side_effect=RuntimeError("boom")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + with pytest.raises(RuntimeError, match="boom"): + await adapter.generate(system="sys", human="user") + span = exporter.get_finished_spans()[0] + events = [e.name for e in span.events] + assert "exception" in events + + async def test_exception_propagates(self, tracer, exporter): + inner = _make_inner(side_effect=ValueError("bad input")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + with pytest.raises(ValueError, match="bad input"): + await adapter.generate(system="sys", human="user") + + async def test_span_still_finished_after_exception(self, tracer, exporter): + inner = _make_inner(side_effect=RuntimeError("fail")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + with pytest.raises(RuntimeError): + await adapter.generate(system="sys", human="user") + assert len(exporter.get_finished_spans()) == 1 + + +# ── Default tracer ──────────────────────────────────────────────────────────── + + +class TestPhoenixTracingAdapterDefaultTracer: + async def test_uses_default_tracer_when_none_provided(self): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner) + result = await adapter.generate(system="sys", human="user") + assert result.content == "The answer is 42." diff --git a/integrations/braintrust-python/.gitignore b/integrations/braintrust-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/braintrust-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/braintrust-python/CHANGELOG.md b/integrations/braintrust-python/CHANGELOG.md new file mode 100644 index 00000000..b0a24abb --- /dev/null +++ b/integrations/braintrust-python/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## [0.1.0] - 2026-06-11 + +### Features + +- Initial release: `BraintrustAnthropicAdapter` and `BraintrustProxyAdapter` diff --git a/integrations/braintrust-python/pyproject.toml b/integrations/braintrust-python/pyproject.toml new file mode 100644 index 00000000..cd699b20 --- /dev/null +++ b/integrations/braintrust-python/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-braintrust-scorers" +version = "0.1.0" +description = "Braintrust adapter for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "braintrust", "evals", "scoring"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + "anthropic>=0.40.0", +] + +[project.optional-dependencies] +braintrust = [ + "braintrust>=0.0.100", +] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/braintrust-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_braintrust_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/braintrust-python/src/learning_commons_braintrust_scorers/__init__.py b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/__init__.py new file mode 100644 index 00000000..5a9664b1 --- /dev/null +++ b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/__init__.py @@ -0,0 +1,28 @@ +"""learning-commons-braintrust-scorers + +Braintrust adapters for Learning Commons evaluators. + +Adapters implement LLMGeneratorProtocol and can be passed directly to any +evaluator that accepts an ``llm_provider`` argument:: + + from learning_commons_braintrust_scorers import BraintrustAnthropicAdapter + from learning_commons_evaluators.evaluators.gla import ( + GradeLevelAppropriatenessEvaluator, + GradeLevelAppropriatenessEvaluationInput, + ) + + evaluator = GradeLevelAppropriatenessEvaluator( + config=..., + llm_provider=BraintrustAnthropicAdapter(project="my-project"), + ) + result = await evaluator.evaluate(GradeLevelAppropriatenessEvaluationInput(text="...")) + print(result.answer.score) # e.g. "6-8" + print(result.explanation.summary) # reasoning text +""" + +from learning_commons_braintrust_scorers.adapter import ( + BraintrustAnthropicAdapter, + BraintrustProxyAdapter, +) + +__all__ = ["BraintrustAnthropicAdapter", "BraintrustProxyAdapter"] diff --git a/integrations/braintrust-python/src/learning_commons_braintrust_scorers/adapter.py b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/adapter.py new file mode 100644 index 00000000..7b4f87cc --- /dev/null +++ b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/adapter.py @@ -0,0 +1,155 @@ +"""Braintrust adapters implementing LLMGeneratorProtocol. + +Two adapters share a common base that handles the Anthropic generation call: + +``BraintrustAnthropicAdapter`` + Uses ``braintrust.auto_instrument()`` to intercept Anthropic SDK calls. + Requires the ``[braintrust]`` optional dependency:: + + pip install learning-commons-braintrust-scorers[braintrust] + + Usage:: + + from learning_commons_braintrust_scorers import BraintrustAnthropicAdapter + + adapter = BraintrustAnthropicAdapter(project="my-project") + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) + +``BraintrustProxyAdapter`` + Routes calls through ``https://api.braintrust.dev/v1/proxy``. + No Braintrust SDK required — only the ``anthropic`` package:: + + pip install learning-commons-braintrust-scorers + + Usage:: + + from learning_commons_braintrust_scorers import BraintrustProxyAdapter + + adapter = BraintrustProxyAdapter(project="my-project") + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) +""" + +from __future__ import annotations + +import anthropic +from anthropic import NOT_GIVEN + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse + +_DEFAULT_MODEL = "claude-opus-4-8-20250514" +_DEFAULT_MAX_TOKENS = 4096 + + +class _AnthropicAdapterBase: + """Shared Anthropic generation logic for Braintrust adapters. + + Subclasses set ``self._client`` and ``self._model`` in ``__init__``. + """ + + _client: anthropic.AsyncAnthropic + _model: str + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + msg = await self._client.messages.create( + model=self._model, + system=system, + messages=[{"role": "user", "content": human}], + max_tokens=config.max_tokens if (config and config.max_tokens is not None) else _DEFAULT_MAX_TOKENS, + # Use NOT_GIVEN (not None) so the field is omitted from the request body. + # Passing None serialises as {"temperature": null} which the Anthropic API rejects. + temperature=config.temperature if (config and config.temperature is not None) else NOT_GIVEN, + ) + # Find the first text block; content may include ThinkingBlock or ToolUseBlock. + text_block = next((b for b in msg.content if b.type == "text"), None) + if text_block is None: + raise ValueError( + f"Anthropic response from {msg.model} contained no text block " + f"(content types: {[b.type for b in msg.content]})" + ) + return LLMResponse( + content=text_block.text, + model=msg.model, + input_tokens=msg.usage.input_tokens, + output_tokens=msg.usage.output_tokens, + ) + + async def aclose(self) -> None: + await self._client.close() + + +class BraintrustAnthropicAdapter(_AnthropicAdapterBase): + """Adapter using Braintrust auto-instrumentation of the Anthropic SDK. + + Calls ``braintrust.auto_instrument()`` at construction time (idempotent). + When ``project`` is provided, calls ``braintrust.init(project=project)`` + so that traces are associated with the correct Braintrust project. + + Requires ``pip install learning-commons-braintrust-scorers[braintrust]``. + + Args: + model: Anthropic model ID. + project: Braintrust project name. When provided, initialises the + Braintrust SDK so traces appear under this project in the UI. + """ + + def __init__( + self, + model: str = _DEFAULT_MODEL, + *, + project: str | None = None, + ) -> None: + import braintrust + + braintrust.auto_instrument() + if project: + braintrust.init(project=project) + self._client = anthropic.AsyncAnthropic() + self._model = model + + +class BraintrustProxyAdapter(_AnthropicAdapterBase): + """Adapter using Braintrust AI Proxy — no Braintrust SDK required. + + Routes Anthropic API calls through ``https://api.braintrust.dev/v1/proxy``. + Braintrust logs all calls automatically via HTTP interception. + + Args: + model: Anthropic model ID. + api_key: Braintrust API key. Falls back to the ``BRAINTRUST_API_KEY`` + environment variable. Raises ``ValueError`` if neither is set + or if the resolved value is blank. + project: Braintrust project name. Passed as the ``x-bt-parent`` header + so traces appear under the correct project in the Braintrust UI. + + Raises: + ValueError: At construction time when no non-blank API key is available. + """ + + def __init__( + self, + model: str = _DEFAULT_MODEL, + *, + api_key: str | None = None, + project: str | None = None, + ) -> None: + import os + + resolved_key = (api_key or os.environ.get("BRAINTRUST_API_KEY") or "").strip() + if not resolved_key: + raise ValueError( + "Braintrust API key is required. Provide it via the api_key argument " + "or set the BRAINTRUST_API_KEY environment variable." + ) + + self._client = anthropic.AsyncAnthropic( + base_url="https://api.braintrust.dev/v1/proxy", + auth_token=resolved_key, + default_headers={"x-bt-parent": f"project_name:{project}"} if project else {}, + ) + self._model = model diff --git a/integrations/braintrust-python/src/learning_commons_braintrust_scorers/py.typed b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/braintrust-python/tests/test_adapter.py b/integrations/braintrust-python/tests/test_adapter.py new file mode 100644 index 00000000..259aaacb --- /dev/null +++ b/integrations/braintrust-python/tests/test_adapter.py @@ -0,0 +1,296 @@ +"""Tests for BraintrustAnthropicAdapter and BraintrustProxyAdapter.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _make_anthropic_message( + text: str = "response text", + model: str = "claude-opus-4-8-20250514", + input_tokens: int = 10, + output_tokens: int = 20, +) -> MagicMock: + msg = MagicMock() + text_block = MagicMock(type="text", text=text) + msg.content = [text_block] + msg.model = model + msg.usage.input_tokens = input_tokens + msg.usage.output_tokens = output_tokens + return msg + + +def _make_async_anthropic_client(message: MagicMock | None = None) -> MagicMock: + """Return a mock AsyncAnthropic client whose messages.create returns *message*.""" + client = MagicMock() + client.messages.create = AsyncMock(return_value=message or _make_anthropic_message()) + client.close = AsyncMock() + return client + + +# ── BraintrustAnthropicAdapter ──────────────────────────────────────────────── + + +class TestBraintrustAnthropicAdapter: + def _make_adapter(self, model: str = "claude-opus-4-8-20250514", project: str | None = None): + """Construct adapter with braintrust and anthropic mocked out.""" + mock_client = _make_async_anthropic_client() + mock_braintrust = MagicMock() + + with ( + patch("braintrust.auto_instrument", mock_braintrust.auto_instrument), + patch.dict("sys.modules", {"braintrust": mock_braintrust}), + patch("anthropic.AsyncAnthropic", return_value=mock_client), + ): + from learning_commons_braintrust_scorers.adapter import BraintrustAnthropicAdapter + + adapter = BraintrustAnthropicAdapter(model=model, project=project) + + # Attach the mock client so tests can make assertions on it. + adapter._client = mock_client + return adapter, mock_client, mock_braintrust + + def test_auto_instrument_called_at_construction(self): + mock_client = _make_async_anthropic_client() + mock_braintrust = MagicMock() + + with ( + patch.dict("sys.modules", {"braintrust": mock_braintrust}), + patch("anthropic.AsyncAnthropic", return_value=mock_client), + ): + from importlib import reload + + import learning_commons_braintrust_scorers.adapter as mod + + reload(mod) + mod.BraintrustAnthropicAdapter() + + # Use .called (not assert_called_once) — reload() inside _make_adapter() may have + # already triggered a call, making assert_called_once() order-dependent across tests. + assert mock_braintrust.auto_instrument.called + + async def test_generate_returns_llm_response(self): + msg = _make_anthropic_message( + text="some output", model="claude-opus-4-8-20250514", input_tokens=5, output_tokens=15 + ) + adapter, mock_client, _ = self._make_adapter() + mock_client.messages.create.return_value = msg + + result = await adapter.generate(system="sys prompt", human="user prompt") + + assert isinstance(result, LLMResponse) + assert result.content == "some output" + assert result.model == "claude-opus-4-8-20250514" + assert result.input_tokens == 5 + assert result.output_tokens == 15 + + async def test_generate_passes_system_and_human(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="the system", human="the human") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["system"] == "the system" + assert call_kwargs["messages"] == [{"role": "user", "content": "the human"}] + + async def test_generate_default_max_tokens(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["max_tokens"] == 4096 + + async def test_generate_default_temperature(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + call_kwargs = mock_client.messages.create.call_args[1] + # When no config is provided, temperature=NOT_GIVEN (field omitted from request) + from anthropic import NOT_GIVEN + assert call_kwargs["temperature"] is NOT_GIVEN + + async def test_generate_respects_config_max_tokens(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=None, max_tokens=512) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["max_tokens"] == 512 + + async def test_generate_respects_config_temperature(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=0.7, max_tokens=None) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["temperature"] == 0.7 + + async def test_generate_config_max_tokens_none_falls_back_to_4096(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=0.5, max_tokens=None) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["max_tokens"] == 4096 + + async def test_generate_uses_configured_model(self): + adapter, mock_client, _ = self._make_adapter(model="claude-haiku-3-5-20251022") + + await adapter.generate(system="s", human="h") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["model"] == "claude-haiku-3-5-20251022" + + async def test_aclose_calls_client_close(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.aclose() + + mock_client.close.assert_called_once() + + +# ── BraintrustProxyAdapter ──────────────────────────────────────────────────── + + +class TestBraintrustProxyAdapter: + def _make_adapter( + self, + model: str = "claude-opus-4-8-20250514", + api_key: str = "test-key", # always provide a key; test ValueError separately + project: str | None = None, + env: dict | None = None, + ): + mock_client = _make_async_anthropic_client() + captured: dict = {} + + def capture_constructor(**kwargs): + captured.update(kwargs) + return mock_client + + extra_env = env or {} + with ( + patch("anthropic.AsyncAnthropic", side_effect=capture_constructor), + patch.dict("os.environ", extra_env, clear=False), + ): + from importlib import reload + + import learning_commons_braintrust_scorers.adapter as mod + + reload(mod) + adapter = mod.BraintrustProxyAdapter(model=model, api_key=api_key, project=project) + + adapter._client = mock_client + return adapter, mock_client, captured + + def test_raises_when_no_api_key(self): + import pytest + from learning_commons_braintrust_scorers.adapter import BraintrustProxyAdapter + with patch.dict("os.environ", {}, clear=True): + with pytest.raises(ValueError, match="API key"): + BraintrustProxyAdapter(api_key=None) + + def test_proxy_base_url(self): + _, _, captured = self._make_adapter() + assert captured["base_url"] == "https://api.braintrust.dev/v1/proxy" + + def test_api_key_from_argument(self): + _, _, captured = self._make_adapter(api_key="my-key") + assert captured["auth_token"] == "my-key" + + def test_api_key_from_env(self): + # Pass api_key=None explicitly so env var is the only source + _, _, captured = self._make_adapter(api_key=None, env={"BRAINTRUST_API_KEY": "env-key"}) + assert captured["auth_token"] == "env-key" + + def test_api_key_argument_takes_precedence_over_env(self): + _, _, captured = self._make_adapter( + api_key="arg-key", env={"BRAINTRUST_API_KEY": "env-key"} + ) + assert captured["auth_token"] == "arg-key" + + def test_project_sets_bt_parent_header(self): + _, _, captured = self._make_adapter(project="my-project") + assert captured["default_headers"] == {"x-bt-parent": "project_name:my-project"} + + def test_no_project_omits_default_headers(self): + _, _, captured = self._make_adapter(project=None) + assert captured.get("default_headers", {}) == {} + + async def test_generate_returns_llm_response(self): + msg = _make_anthropic_message( + text="proxy output", model="claude-opus-4-8-20250514", input_tokens=8, output_tokens=12 + ) + adapter, mock_client, _ = self._make_adapter() + mock_client.messages.create.return_value = msg + + result = await adapter.generate(system="sys", human="usr") + + assert isinstance(result, LLMResponse) + assert result.content == "proxy output" + assert result.input_tokens == 8 + assert result.output_tokens == 12 + + async def test_generate_passes_system_and_human(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="the system", human="the human") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["system"] == "the system" + assert call_kwargs["messages"] == [{"role": "user", "content": "the human"}] + + async def test_generate_default_max_tokens(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + assert mock_client.messages.create.call_args[1]["max_tokens"] == 4096 + + async def test_generate_default_temperature(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + from anthropic import NOT_GIVEN + assert mock_client.messages.create.call_args[1]["temperature"] is NOT_GIVEN + + async def test_generate_respects_config(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=0.1, max_tokens=256) + + await adapter.generate(system="s", human="h", config=config) + + kwargs = mock_client.messages.create.call_args[1] + assert kwargs["max_tokens"] == 256 + assert kwargs["temperature"] == 0.1 + + async def test_generate_uses_configured_model(self): + adapter, mock_client, _ = self._make_adapter(model="claude-haiku-3-5-20251022") + + await adapter.generate(system="s", human="h") + + assert mock_client.messages.create.call_args[1]["model"] == "claude-haiku-3-5-20251022" + + async def test_aclose_calls_client_close(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.aclose() + + mock_client.close.assert_called_once() + + +# ── Protocol conformance ────────────────────────────────────────────────────── + + diff --git a/integrations/inspect-python/.gitignore b/integrations/inspect-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/inspect-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/inspect-python/CHANGELOG.md b/integrations/inspect-python/CHANGELOG.md new file mode 100644 index 00000000..825c32f0 --- /dev/null +++ b/integrations/inspect-python/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/integrations/inspect-python/README.md b/integrations/inspect-python/README.md new file mode 100644 index 00000000..0195d3c1 --- /dev/null +++ b/integrations/inspect-python/README.md @@ -0,0 +1,79 @@ +# learning-commons-inspect-scorers + +[Inspect AI](https://inspect.aisi.org.uk/) scorer wrappers for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +## Installation + +```bash +pip install learning-commons-inspect-scorers +``` + +> **Note:** Requires `learning-commons-evaluators>=0.2.0`. During local development +> (before 0.2.0 is published), install the SDK from the repo root first: +> ```bash +> pip install -e sdks/python +> pip install -e integrations/inspect-python +> ``` + +## Usage + +### Grade Level Appropriateness scorer + +Evaluates whether model output (or generated artifact files) is written at the +appropriate reading level for a target K-12 grade band. + +```python +from inspect_ai import Task, task +from inspect_ai.dataset import csv_dataset, FieldSpec +from inspect_ai.solver import generate +from learning_commons_inspect_scorers import gla_scorer +from learning_commons_evaluators.config import create_config_no_telemetry +from learning_commons_evaluators.schemas.config import GoogleLLMProviderConfig + +config = create_config_no_telemetry( + google_llm_provider_config=GoogleLLMProviderConfig(api_key="your-key"), +) + +@task +def my_eval(): + return Task( + dataset=csv_dataset("samples.csv"), # requires target_grade column + solver=[generate()], + scorer=gla_scorer(config=config), + ) +``` + +The dataset CSV must include a `target_grade` metadata column with one of: +`K-1`, `2-3`, `4-5`, `6-8`, `9-10`, `11-CCR`. + +### Scoring artifact files (edu-panda-skill-harness) + +```python +scorer=gla_scorer(config=config, text_source="artifacts") +``` + +### Re-scoring an existing log from the CLI + +Once installed, scorers are registered via Inspect's entry point system: + +```bash +inspect score logs/my-eval.eval --scorer learning_commons_inspect_scorers/gla_scorer +``` + +## Configuration + +| Parameter | Default | Description | +|---|---|---| +| `config` | env vars | `EvaluatorConfig`. If `None`, reads `GOOGLE_API_KEY`, `ANTHROPIC_API_KEY`, or `OPENAI_API_KEY` from the environment. | +| `text_source` | `"completion"` | `"completion"` scores `state.output.completion`; `"artifacts"` joins `state.metadata["artifacts"]` file contents. | +| `target_grade_key` | `"target_grade"` | Metadata key holding the expected grade band. | +| `allow_adjacent` | `True` | If `True`, the one grade band above or below the target also passes. | + +## Development + +```bash +# From repo root +pip install -e sdks/python +pip install -e "integrations/inspect-python[dev]" +pytest integrations/inspect-python/tests/ +``` diff --git a/integrations/inspect-python/pyproject.toml b/integrations/inspect-python/pyproject.toml new file mode 100644 index 00000000..8ec39cb4 --- /dev/null +++ b/integrations/inspect-python/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-inspect-scorers" +version = "0.1.0" +description = "Inspect AI scorer wrappers for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "inspect", "evals", "scoring"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + "inspect-ai>=0.3.2", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/inspect-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +# Registers scorers with Inspect's component discovery via setuptools entry points. +# Once installed, scorers are accessible as e.g. `learning_commons_inspect_scorers/gla_scorer` +# from the CLI: inspect score log.eval --scorer learning_commons_inspect_scorers/gla_scorer +[project.entry-points.inspect_ai] +learning_commons_inspect_scorers = "learning_commons_inspect_scorers._registry" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_inspect_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +plugins = ["pydantic.mypy"] +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py new file mode 100644 index 00000000..c9f2f977 --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py @@ -0,0 +1,6 @@ +"""Learning Commons Inspect scorers — Inspect AI wrappers for LC evaluators.""" + +from learning_commons_inspect_scorers.adapter import InspectModelAdapter +from learning_commons_inspect_scorers.gla import gla_scorer + +__all__ = ["InspectModelAdapter", "gla_scorer"] diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py new file mode 100644 index 00000000..7e353db4 --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py @@ -0,0 +1,8 @@ +"""Entry point registry — imported by Inspect via the inspect_ai setuptools entry point. + +Importing this module registers all scorers with Inspect's component system, +making them accessible by name (e.g. learning_commons_inspect_scorers/gla_scorer) +from both the Python API and the CLI. +""" + +from learning_commons_inspect_scorers.gla import gla_scorer # noqa: F401 — import triggers @scorer registry side-effect diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py new file mode 100644 index 00000000..448355f2 --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py @@ -0,0 +1,70 @@ +"""Inspect AI model adapter implementing LLMGeneratorProtocol. + +This is a separate package (``learning-commons-inspect-scorers``) rather than +part of ``learning-commons-evaluators`` because it introduces ``inspect-ai`` as +a hard dependency — a heavy framework that not all SDK users need. + +**Versioning contract**: this package requires ``learning-commons-evaluators>=0.2.0`` +where ``LLMGeneratorProtocol`` was introduced. If a new method is added to the +protocol, bump the lower bound here and update this adapter. + +**Building a new integration** (e.g. ``integrations/langsmith-python``): +implement ``LLMGeneratorProtocol`` — a single async ``generate()`` method that +calls your framework's model and returns ``LLMResponse`` — then inject it into +any evaluator via ``GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter)``. +""" + +from __future__ import annotations + +from inspect_ai.model import ( + ChatMessageSystem, + ChatMessageUser, + GenerateConfig as InspectGenConfig, + get_model, +) + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse + + +class InspectModelAdapter: + """Wraps Inspect's get_model() to satisfy LLMGeneratorProtocol. + + Pass a model string in the same form accepted by Inspect's ``--model`` flag, + for example ``"anthropic/claude-opus-4-8"`` or ``"openai/gpt-4o"``. + + Example:: + + adapter = InspectModelAdapter("anthropic/claude-opus-4-8") + evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, + ) + """ + + def __init__(self, model_name: str) -> None: + self._model_name = model_name + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + # get_model() is memoized by Inspect — repeated calls with the same string + # return the cached Model object without reconstruction. + inspect_model = get_model(self._model_name) + inspect_config = InspectGenConfig( + temperature=config.temperature if config is not None else None, + max_tokens=config.max_tokens if config is not None else None, + ) + output = await inspect_model.generate( + [ChatMessageSystem(content=system), ChatMessageUser(content=human)], + config=inspect_config, + ) + return LLMResponse( + content=output.completion, + model=self._model_name, + input_tokens=getattr(output.usage, "input_tokens", None), + output_tokens=getattr(output.usage, "output_tokens", None), + ) diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py new file mode 100644 index 00000000..5c0f1b90 --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py @@ -0,0 +1,151 @@ +"""Inspect scorer wrapper for the Grade Level Appropriateness evaluator.""" + +from __future__ import annotations + +from pathlib import Path + +from inspect_ai.scorer import CORRECT, INCORRECT, Score, Target, accuracy, scorer +from inspect_ai.solver import TaskState + +from learning_commons_evaluators.config import create_config_no_telemetry +from learning_commons_evaluators.evaluators.grade_level_appropriateness import ( + GradeLevelAppropriatenessEvaluationInput, + GradeLevelAppropriatenessEvaluator, +) +from learning_commons_evaluators.schemas.errors import ( + APIError, + ConfigurationError, + InputValidationError, + OutputValidationError, +) +from learning_commons_evaluators.schemas.grade_level_appropriateness import GradeLevelAnswer + +from learning_commons_inspect_scorers.adapter import InspectModelAdapter + +GRADE_BANDS = [m.score for m in GradeLevelAnswer] +MAX_ARTIFACT_CHARS = 20_000 +MAX_TOTAL_ARTIFACT_CHARS = 90_000 # safely under the GLA evaluator's 100k limit + + +def _completion_text(state: TaskState) -> str: + return (state.output.completion or "")[:MAX_ARTIFACT_CHARS] + + +def _artifact_text(state: TaskState) -> str: + artifacts = state.metadata.get("artifacts", []) + parts = [] + for a in artifacts: + path_str = a.get("path", "") + filename = a.get("filename", "unknown") + try: + content = Path(path_str).read_text(encoding="utf-8")[:MAX_ARTIFACT_CHARS] + parts.append(f"### {filename}\n\n{content}") + except Exception: + # Skip unreadable artifacts rather than feeding error strings to the LLM. + pass + joined = "\n\n---\n\n".join(parts) if parts else "" + return joined[:MAX_TOTAL_ARTIFACT_CHARS] + + +def _acceptable_bands(target: str, allow_adjacent: bool) -> set[str]: + idx = GRADE_BANDS.index(target) if target in GRADE_BANDS else -1 + if idx == -1 or not allow_adjacent: + return {target} + return {GRADE_BANDS[i] for i in range(max(0, idx - 1), min(len(GRADE_BANDS), idx + 2))} + + +@scorer(metrics=[accuracy()]) +def gla_scorer( + grader_model: str = "anthropic/claude-opus-4-8", + text_source: str = "completion", + target_grade_key: str = "target_grade", + allow_adjacent: bool = True, +): + """Score output for grade-level appropriateness against a target grade band. + + Uses Inspect's active model (via InspectModelAdapter) to run the GLA evaluator. + No separate LLM API keys are required — the model is resolved through Inspect's + own model configuration. + + Args: + grader_model: Inspect model string for the grading LLM. + Default: ``"anthropic/claude-opus-4-8"``. + text_source: Where to read the text to evaluate. Must be ``"completion"`` + (default, uses ``state.output.completion``) or ``"artifacts"`` + (concatenates ``state.metadata["artifacts"]`` file contents). + target_grade_key: Metadata key holding the expected grade band string. + Default: ``"target_grade"``. Must be one of: K-1, 2-3, + 4-5, 6-8, 9-10, 11-CCR. + allow_adjacent: If ``True`` (default), the one grade band above or below + the target also counts as a pass. Set to ``False`` for an + exact match. + + Returns ``Score.unscored()`` when ``target_grade_key`` is absent or not a valid + grade band, when no text is available to evaluate, or when a transient API/parse + error occurs. Re-raises ``ConfigurationError`` and ``InputValidationError`` as + task-level failures. + + ``Score.value`` is ``CORRECT`` (pass) or ``INCORRECT`` (fail). + ``Score.metadata`` contains ``gla_grade``, ``target_grade``, ``alternative_grade``, + and ``scaffolding_needed``. + """ + if text_source not in ("completion", "artifacts"): + raise ValueError(f"text_source must be 'completion' or 'artifacts', got {text_source!r}") + + adapter = InspectModelAdapter(grader_model) + # config is required by BaseEvaluator but no API keys are read here — + # the llm_provider bypasses the LangChain provider path entirely. + config = create_config_no_telemetry() + evaluator = GradeLevelAppropriatenessEvaluator(config=config, llm_provider=adapter) + get_text = _artifact_text if text_source == "artifacts" else _completion_text + + async def score(state: TaskState, target: Target) -> Score | None: + target_grade: str = (state.metadata.get(target_grade_key) or "").strip() + if not target_grade or target_grade not in GRADE_BANDS: + return Score.unscored( + explanation=( + f"{target_grade_key!r} is missing or not a valid grade band. " + f"Valid bands: {', '.join(GRADE_BANDS)}" + ), + metadata={"target_grade_key": target_grade_key, "target_grade": target_grade or None}, + ) + + text = get_text(state) + if not text.strip(): + return Score.unscored( + explanation="No text to evaluate (empty completion or no readable artifacts).", + metadata={"target_grade": target_grade, "gla_grade": None}, + ) + + try: + result = await evaluator.evaluate( + GradeLevelAppropriatenessEvaluationInput(text=text) + ) + except (ConfigurationError, InputValidationError): + raise # setup/programming errors — let Inspect surface them as task failures + except APIError as exc: # OutputValidationError is a subclass of APIError + return Score.unscored( + explanation=f"GLA evaluation failed: {exc}", + metadata={"target_grade": target_grade, "gla_grade": None}, + ) + + gla_grade: str = result.answer.score # e.g. "6-8" + passed = gla_grade in _acceptable_bands(target_grade, allow_adjacent) + + return Score( + value=CORRECT if passed else INCORRECT, + answer=gla_grade, + explanation=( + f"Target: {target_grade} | GLA: {gla_grade} | " + f"{'PASS' if passed else 'FAIL'}\n" + + (result.explanation.summary or "") + ), + metadata={ + "gla_grade": gla_grade, + "target_grade": target_grade, + "alternative_grade": result.explanation.details["alternative_grade"], + "scaffolding_needed": result.explanation.details["scaffolding_needed"], + }, + ) + + return score diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed b/integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/inspect-python/tests/test_gla_scorer.py b/integrations/inspect-python/tests/test_gla_scorer.py new file mode 100644 index 00000000..5d7edd63 --- /dev/null +++ b/integrations/inspect-python/tests/test_gla_scorer.py @@ -0,0 +1,341 @@ +"""Tests for the GLA Inspect scorer wrapper.""" + +from __future__ import annotations + +import math +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from inspect_ai.scorer import CORRECT, INCORRECT + +from learning_commons_evaluators.schemas.errors import APIError, ConfigurationError +from learning_commons_inspect_scorers.gla import _acceptable_bands, gla_scorer + +GRADE_BANDS = ["K-1", "2-3", "4-5", "6-8", "9-10", "11-CCR"] + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def _make_state(completion: str = "", metadata: dict | None = None) -> MagicMock: + state = MagicMock() + state.output.completion = completion + state.metadata = metadata or {} + return state + + +def _make_target() -> MagicMock: + return MagicMock() + + +def _make_gla_result(grade: str = "6-8") -> MagicMock: + result = MagicMock() + result.answer.score = grade + result.explanation.summary = "Test reasoning." + result.explanation.details = { + "alternative_grade": "4-5", + "scaffolding_needed": "Pre-teach vocabulary.", + } + return result + + +def _make_scorer(grade_result: str = "6-8", side_effect=None, **scorer_kwargs): + """Build a gla_scorer with a patched evaluator. + + Patches GradeLevelAppropriatenessEvaluator so no real LLM calls are made. + The evaluator instance captured in the scorer closure is the mock, so calls + work normally after construction. + """ + mock_evaluator = MagicMock() + mock_evaluator.evaluate = AsyncMock( + return_value=_make_gla_result(grade_result), + side_effect=side_effect, + ) + with patch( + "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", + return_value=mock_evaluator, + ): + scorer_fn = gla_scorer(**scorer_kwargs) + # The scorer closure already holds the mock_evaluator instance — no further + # patching needed for subsequent calls. + return scorer_fn, mock_evaluator + + +# ── _acceptable_bands ──────────────────────────────────────────────────────── + + +class TestAcceptableBands: + def test_middle_grade_allow_adjacent(self): + assert _acceptable_bands("6-8", allow_adjacent=True) == {"4-5", "6-8", "9-10"} + + def test_lower_boundary_allow_adjacent(self): + # K-1 is at index 0 — no band below it + assert _acceptable_bands("K-1", allow_adjacent=True) == {"K-1", "2-3"} + + def test_upper_boundary_allow_adjacent(self): + # 11-CCR is at end — no band above it + assert _acceptable_bands("11-CCR", allow_adjacent=True) == {"9-10", "11-CCR"} + + def test_exact_match_only(self): + assert _acceptable_bands("6-8", allow_adjacent=False) == {"6-8"} + + def test_invalid_grade_returns_singleton(self): + assert _acceptable_bands("invalid", allow_adjacent=True) == {"invalid"} + + @pytest.mark.parametrize("band", GRADE_BANDS) + def test_all_bands_are_valid(self, band): + result = _acceptable_bands(band, allow_adjacent=True) + assert band in result + assert len(result) >= 1 + + +# ── gla_scorer — factory validation ────────────────────────────────────────── + + +class TestGlaScorerFactory: + def test_invalid_text_source_raises(self): + with pytest.raises(ValueError, match="text_source must be"): + gla_scorer(text_source="html") + + def test_valid_text_sources_accepted(self): + for src in ("completion", "artifacts"): + with patch("learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator"): + gla_scorer(text_source=src) # must not raise + + +# ── gla_scorer — score routing ──────────────────────────────────────────────── + + +class TestGlaScorer: + async def test_matching_grade_returns_correct(self): + scorer_fn, _ = _make_scorer(grade_result="6-8") + state = _make_state(completion="Sample text.", metadata={"target_grade": "6-8"}) + score = await scorer_fn(state, _make_target()) + assert score.value == CORRECT + assert score.answer == "6-8" + + async def test_adjacent_grade_passes_when_allow_adjacent(self): + scorer_fn, _ = _make_scorer(grade_result="4-5") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_non_adjacent_grade_fails(self): + scorer_fn, _ = _make_scorer(grade_result="K-1") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == INCORRECT + + async def test_exact_match_only_when_adjacent_disabled(self): + scorer_fn, _ = _make_scorer(grade_result="4-5", allow_adjacent=False) + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == INCORRECT + + async def test_boundary_k1_adjacent_passes(self): + scorer_fn, _ = _make_scorer(grade_result="2-3") + state = _make_state(completion="Sample.", metadata={"target_grade": "K-1"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_boundary_11ccr_adjacent_passes(self): + scorer_fn, _ = _make_scorer(grade_result="9-10") + state = _make_state(completion="Sample.", metadata={"target_grade": "11-CCR"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_missing_target_grade_returns_unscored(self): + scorer_fn, _ = _make_scorer() + state = _make_state(completion="Sample.", metadata={}) + score = await scorer_fn(state, _make_target()) + assert math.isnan(float(score.value)) + + async def test_invalid_target_grade_returns_unscored(self): + scorer_fn, _ = _make_scorer() + state = _make_state(completion="Sample.", metadata={"target_grade": "Grade 5"}) + score = await scorer_fn(state, _make_target()) + assert math.isnan(float(score.value)) + + async def test_empty_completion_returns_unscored(self): + scorer_fn, mock_evaluator = _make_scorer() + state = _make_state(completion="", metadata={"target_grade": "6-8"}) + score = await scorer_fn(state, _make_target()) + assert math.isnan(float(score.value)) + mock_evaluator.evaluate.assert_not_called() + + async def test_api_error_returns_unscored(self): + scorer_fn, _ = _make_scorer(side_effect=APIError("rate limit")) + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + score = await scorer_fn(state, _make_target()) + assert math.isnan(float(score.value)) + assert "rate limit" in score.explanation + + async def test_configuration_error_propagates(self): + scorer_fn, _ = _make_scorer(side_effect=ConfigurationError("no key")) + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + with pytest.raises(ConfigurationError): + await scorer_fn(state, _make_target()) + + async def test_score_metadata_populated(self): + scorer_fn, _ = _make_scorer(grade_result="6-8") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + score = await scorer_fn(state, _make_target()) + assert score.metadata["gla_grade"] == "6-8" + assert score.metadata["target_grade"] == "6-8" + assert score.metadata["alternative_grade"] == "4-5" + assert score.metadata["scaffolding_needed"] == "Pre-teach vocabulary." + + async def test_custom_target_grade_key(self): + scorer_fn, _ = _make_scorer(grade_result="6-8", target_grade_key="expected_grade") + state = _make_state(completion="Sample.", metadata={"expected_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_custom_target_grade_key_absent_returns_unscored(self): + scorer_fn, _ = _make_scorer(target_grade_key="expected_grade") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + score = await scorer_fn(state, _make_target()) + assert math.isnan(float(score.value)) + + async def test_custom_grader_model_passed_to_adapter(self): + with patch( + "learning_commons_inspect_scorers.gla.InspectModelAdapter" + ) as mock_adapter_cls, patch( + "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator" + ): + gla_scorer(grader_model="openai/gpt-4o") + mock_adapter_cls.assert_called_once_with("openai/gpt-4o") + + +# ── gla_scorer — artifacts text source ──────────────────────────────────────── + + +class TestGlaScorerArtifacts: + async def test_reads_artifact_files(self, tmp_path: Path): + artifact = tmp_path / "lesson.md" + artifact.write_text("The mitochondria is the powerhouse of the cell.") + + scorer_fn, mock_evaluator = _make_scorer( + grade_result="6-8", text_source="artifacts" + ) + state = _make_state( + metadata={ + "target_grade": "6-8", + "artifacts": [{"path": str(artifact), "filename": "lesson.md"}], + } + ) + score = await scorer_fn(state, _make_target()) + assert score.value == CORRECT + called_text = mock_evaluator.evaluate.call_args[0][0].text.value + assert "mitochondria" in called_text + + async def test_unreadable_artifact_returns_unscored(self): + scorer_fn, mock_evaluator = _make_scorer(text_source="artifacts") + state = _make_state( + metadata={ + "target_grade": "6-8", + "artifacts": [{"path": "/nonexistent/path.md", "filename": "missing.md"}], + } + ) + score = await scorer_fn(state, _make_target()) + assert math.isnan(float(score.value)) + mock_evaluator.evaluate.assert_not_called() + + async def test_multiple_artifacts_joined_with_separator(self, tmp_path: Path): + f1 = tmp_path / "a.md" + f2 = tmp_path / "b.md" + f1.write_text("First document.") + f2.write_text("Second document.") + + scorer_fn, mock_evaluator = _make_scorer( + grade_result="4-5", text_source="artifacts" + ) + state = _make_state( + metadata={ + "target_grade": "4-5", + "artifacts": [ + {"path": str(f1), "filename": "a.md"}, + {"path": str(f2), "filename": "b.md"}, + ], + } + ) + await scorer_fn(state, _make_target()) + called_text = mock_evaluator.evaluate.call_args[0][0].text.value + assert "First document." in called_text + assert "Second document." in called_text + assert "---" in called_text + + +# ── Integration tests (mockllm/model) ──────────────────────────────────────── + + +class TestGlaScorerIntegration: + """End-to-end tests using Inspect's built-in mockllm/model provider. + + These tests validate that gla_scorer satisfies the Inspect Scorer protocol + and wires correctly through eval() — without making any real LLM calls. + The GLA evaluator itself is still mocked at the SDK boundary. + + .. note:: + These tests are synchronous (``def``, not ``async def``) because + ``inspect_ai.eval()`` calls ``anyio.run()`` internally to start its own + event loop. Using ``async def`` under ``pytest-asyncio asyncio_mode=auto`` + would start a second event loop, causing an anyio ``ScopeMismatch`` error. + """ + + def test_scorer_wires_through_eval(self): + from unittest.mock import AsyncMock, patch + + from inspect_ai import Task, eval + from inspect_ai.dataset import Sample + from inspect_ai.solver import generate + + mock_evaluator = MagicMock() + mock_evaluator.evaluate = AsyncMock(return_value=_make_gla_result("6-8")) + + with patch( + "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", + return_value=mock_evaluator, + ): + scorer = gla_scorer() + + task = Task( + dataset=[ + Sample( + input="Write a short paragraph for 6th graders.", + metadata={"target_grade": "6-8"}, + ) + ], + solver=[generate()], + scorer=scorer, + ) + + log = eval(task, model="mockllm/model")[0] + + assert log.status == "success" + sample = log.samples[0] + assert sample.score is not None + assert sample.score.value == CORRECT + + def test_unscored_sample_does_not_count_as_failure(self): + from unittest.mock import AsyncMock, patch + + from inspect_ai import Task, eval + from inspect_ai.dataset import Sample + from inspect_ai.solver import generate + + mock_evaluator = MagicMock() + mock_evaluator.evaluate = AsyncMock(return_value=_make_gla_result("6-8")) + + with patch( + "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", + return_value=mock_evaluator, + ): + scorer = gla_scorer() + + # Sample has no target_grade — should be unscored, not INCORRECT + task = Task( + dataset=[Sample(input="Write something.", metadata={})], + solver=[generate()], + scorer=scorer, + ) + + log = eval(task, model="mockllm/model")[0] + sample = log.samples[0] + assert sample.score is not None + assert math.isnan(float(sample.score.value)) diff --git a/integrations/langfuse-python/.gitignore b/integrations/langfuse-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/langfuse-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/langfuse-python/CHANGELOG.md b/integrations/langfuse-python/CHANGELOG.md new file mode 100644 index 00000000..825c32f0 --- /dev/null +++ b/integrations/langfuse-python/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/integrations/langfuse-python/pyproject.toml b/integrations/langfuse-python/pyproject.toml new file mode 100644 index 00000000..68cc72e8 --- /dev/null +++ b/integrations/langfuse-python/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-langfuse-scorers" +version = "0.1.0" +description = "Langfuse tracing adapter for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "langfuse", "tracing", "observability"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + # Langfuse v3+ (released 2025) removed trace()/generation() in favour of an + # OTel-based API. Pin to v2 until this adapter is migrated to start_as_current_generation(). + # TODO: migrate to v3+ OTel pattern and remove the upper bound. + "langfuse>=2.0.0,<3.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/langfuse-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_langfuse_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/__init__.py b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/__init__.py new file mode 100644 index 00000000..6e9fa653 --- /dev/null +++ b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/__init__.py @@ -0,0 +1,5 @@ +"""Learning Commons Langfuse scorers — Langfuse tracing adapter for LC evaluators.""" + +from learning_commons_langfuse_scorers.adapter import LangfuseTracingAdapter + +__all__ = ["LangfuseTracingAdapter"] diff --git a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py new file mode 100644 index 00000000..ec191ec8 --- /dev/null +++ b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py @@ -0,0 +1,96 @@ +"""LangfuseTracingAdapter — decorator that wraps any LLMGeneratorProtocol and records Langfuse generations. + +.. note:: + This adapter targets the Langfuse v2 SDK (``langfuse>=2.0.0,<3.0.0``). + Langfuse v3+ replaced ``trace()``/``generation()`` with an OTel-based API + (``start_as_current_generation()``). A migration is tracked as a TODO. + +Usage:: + + from learning_commons_langfuse_scorers import LangfuseTracingAdapter + from learning_commons_inspect_scorers.adapter import InspectModelAdapter + + adapter = LangfuseTracingAdapter(InspectModelAdapter("anthropic/claude-opus-4-8")) + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) +""" + +from __future__ import annotations + +import asyncio + +from langfuse import Langfuse + +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class LangfuseTracingAdapter: + """Decorator adapter: wraps any LLMGeneratorProtocol, records Langfuse generations. + + Args: + inner: The underlying adapter to delegate generation to. + langfuse: Langfuse client instance. Defaults to ``Langfuse()``, which + reads ``LANGFUSE_PUBLIC_KEY``, ``LANGFUSE_SECRET_KEY``, and + ``LANGFUSE_HOST`` from the environment. + trace_name: Name for the Langfuse trace. Default: ``"lc_eval"``. + + .. note:: + Each call to ``generate()`` creates a new Langfuse trace. For single-step + evaluators (GLA, conventionality) this produces one trace per evaluation. + For multi-step evaluators (vocabulary: 2 steps), this produces one trace + per LLM call — the steps appear as separate traces rather than nested + generations on a single trace. Pass a unique ``trace_name`` per evaluation + run (e.g. a UUID) if you want to group them by name in the Langfuse UI. + """ + + def __init__( + self, + inner: LLMGeneratorProtocol, + langfuse: Langfuse | None = None, + trace_name: str = "lc_eval", + ) -> None: + self._inner = inner + self._langfuse = langfuse or Langfuse() + self._trace_name = trace_name + + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + lf_trace = self._langfuse.trace(name=self._trace_name) + generation = lf_trace.generation( + name="llm_generate", + input=[ + {"role": "system", "content": system}, + {"role": "user", "content": human}, + ], + model_parameters={ + k: v for k, v in { + "temperature": config.temperature if config else None, + "max_tokens": config.max_tokens if config else None, + }.items() if v is not None + }, + ) + try: + response = await self._inner.generate(system=system, human=human, config=config) + generation.end( + output=response.content, + model=response.model, + usage={ + k: v for k, v in { + "input": response.input_tokens, + "output": response.output_tokens, + }.items() if v is not None + }, + ) + return response + except Exception as exc: + generation.end(level="ERROR", status_message=str(exc)) + raise + + async def aclose(self) -> None: + """Flush buffered Langfuse events. Offloads the blocking flush to a thread pool.""" + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, self._langfuse.flush) diff --git a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/py.typed b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/langfuse-python/tests/test_adapter.py b/integrations/langfuse-python/tests/test_adapter.py new file mode 100644 index 00000000..1a852c56 --- /dev/null +++ b/integrations/langfuse-python/tests/test_adapter.py @@ -0,0 +1,181 @@ +"""Tests for LangfuseTracingAdapter.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse +from learning_commons_langfuse_scorers import LangfuseTracingAdapter + + +def _make_mock_langfuse() -> MagicMock: + """Return a Langfuse mock with the trace/generation chain wired up.""" + mock_generation = MagicMock() + mock_trace = MagicMock() + mock_trace.generation.return_value = mock_generation + mock_langfuse = MagicMock() + mock_langfuse.trace.return_value = mock_trace + return mock_langfuse + + +def _make_inner(response: LLMResponse | None = None, side_effect=None) -> MagicMock: + if response is None: + response = LLMResponse( + content='{"score": "6-8"}', + model="claude-opus-4-8", + input_tokens=10, + output_tokens=20, + ) + inner = MagicMock() + inner.generate = AsyncMock(return_value=response, side_effect=side_effect) + return inner + + +class TestLangfuseTracingAdapterInit: + def test_creates_langfuse_when_not_provided(self): + inner = _make_inner() + with patch("learning_commons_langfuse_scorers.adapter.Langfuse") as mock_cls: + mock_cls.return_value = MagicMock() + adapter = LangfuseTracingAdapter(inner) + mock_cls.assert_called_once_with() + assert adapter._inner is inner + + def test_uses_provided_langfuse_instance(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + assert adapter._langfuse is mock_langfuse + + def test_default_trace_name(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + assert adapter._trace_name == "lc_eval" + + def test_custom_trace_name(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse, trace_name="my_eval") + assert adapter._trace_name == "my_eval" + + +class TestLangfuseTracingAdapterGenerate: + async def test_creates_trace_with_configured_name(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse, trace_name="my_trace") + + await adapter.generate(system="You are a grader.", human="Assess this text.") + + mock_langfuse.trace.assert_called_once_with(name="my_trace") + + async def test_creates_generation_with_correct_input(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + mock_trace = mock_langfuse.trace.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.generate(system="sys prompt", human="user prompt") + + mock_trace.generation.assert_called_once() + call_kwargs = mock_trace.generation.call_args[1] + assert call_kwargs["name"] == "llm_generate" + assert call_kwargs["input"] == [ + {"role": "system", "content": "sys prompt"}, + {"role": "user", "content": "user prompt"}, + ] + + async def test_passes_config_model_parameters(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + mock_trace = mock_langfuse.trace.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + config = GenerateConfig(temperature=0.7, max_tokens=256) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_trace.generation.call_args[1] + assert call_kwargs["model_parameters"] == {"temperature": 0.7, "max_tokens": 256} + + async def test_none_config_sends_none_parameters(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + mock_trace = mock_langfuse.trace.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.generate(system="s", human="h", config=None) + + call_kwargs = mock_trace.generation.call_args[1] + # None values are filtered out — model_parameters is empty when config is None + assert call_kwargs["model_parameters"] == {} + + async def test_calls_inner_generate_with_correct_args(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + config = GenerateConfig(temperature=0.0, max_tokens=128) + + await adapter.generate(system="sys", human="usr", config=config) + + inner.generate.assert_called_once_with(system="sys", human="usr", config=config) + + async def test_returns_inner_response(self): + response = LLMResponse( + content="result", model="gpt-4", input_tokens=5, output_tokens=15 + ) + inner = _make_inner(response=response) + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + result = await adapter.generate(system="s", human="h") + + assert result is response + + async def test_ends_generation_with_response_data(self): + response = LLMResponse( + content="the answer", model="claude-opus-4-8", input_tokens=10, output_tokens=20 + ) + inner = _make_inner(response=response) + mock_langfuse = _make_mock_langfuse() + mock_generation = mock_langfuse.trace.return_value.generation.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.generate(system="s", human="h") + + mock_generation.end.assert_called_once_with( + output="the answer", + model="claude-opus-4-8", + usage={"input": 10, "output": 20}, + ) + + async def test_ends_generation_with_error_on_exception(self): + inner = _make_inner(side_effect=RuntimeError("timeout")) + mock_langfuse = _make_mock_langfuse() + mock_generation = mock_langfuse.trace.return_value.generation.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + with pytest.raises(RuntimeError, match="timeout"): + await adapter.generate(system="s", human="h") + + mock_generation.end.assert_called_once_with(level="ERROR", status_message="timeout") + + async def test_re_raises_exception_after_recording(self): + inner = _make_inner(side_effect=ValueError("bad input")) + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + with pytest.raises(ValueError, match="bad input"): + await adapter.generate(system="s", human="h") + + +class TestLangfuseTracingAdapterAclose: + async def test_aclose_flushes_langfuse(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.aclose() + + mock_langfuse.flush.assert_called_once_with() diff --git a/release-please-config.json b/release-please-config.json index a0cd7202..874ea81a 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -63,6 +63,38 @@ "release-type": "node", "changelog-path": "CHANGELOG.md", "component": "sdks-typescript" + }, + "integrations/inspect-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-inspect-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true + }, + "integrations/langfuse-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-langfuse-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true + }, + "integrations/arize-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-arize-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true + }, + "integrations/braintrust-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-braintrust-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true } } } From 9a59eed73199b3a7c4bbee7795f3206dad703c12 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 22:07:46 -0700 Subject: [PATCH 08/10] fix: address Copilot review findings on integration packages - Remove unused OutputValidationError import from gla.py (F401 lint) - Fix README examples: gla_scorer() takes grader_model not config - Add README.md for arize-python, langfuse-python, braintrust-python - Fix braintrust test: remove patch('braintrust.auto_instrument') which triggered ModuleNotFoundError before sys.modules mock was applied --- integrations/arize-python/README.md | 45 ++++++++++++++ integrations/braintrust-python/README.md | 61 +++++++++++++++++++ .../braintrust-python/tests/test_adapter.py | 4 +- integrations/inspect-python/README.md | 14 ++--- .../learning_commons_inspect_scorers/gla.py | 1 - integrations/langfuse-python/README.md | 50 +++++++++++++++ 6 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 integrations/arize-python/README.md create mode 100644 integrations/braintrust-python/README.md create mode 100644 integrations/langfuse-python/README.md diff --git a/integrations/arize-python/README.md b/integrations/arize-python/README.md new file mode 100644 index 00000000..e724f98e --- /dev/null +++ b/integrations/arize-python/README.md @@ -0,0 +1,45 @@ +# learning-commons-arize-scorers + +[Arize/Phoenix](https://phoenix.arize.com/) OTel tracing adapter for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +Wraps any `LLMGeneratorProtocol` adapter and emits [OpenInference](https://github.com/Arize-ai/openinference) spans compatible with Arize Phoenix and any OTel backend. + +## Installation + +```bash +pip install learning-commons-arize-scorers +``` + +## Usage + +```python +from learning_commons_arize_scorers import PhoenixTracingAdapter +from learning_commons_inspect_scorers.adapter import InspectModelAdapter +from learning_commons_evaluators import GradeLevelAppropriatenessEvaluator +from learning_commons_evaluators.config import create_config_no_telemetry + +adapter = PhoenixTracingAdapter( + InspectModelAdapter("anthropic/claude-opus-4-8"), + capture_message_content=False, # False by default — K-12 privacy +) +evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, +) +``` + +## Configuration + +| Parameter | Default | Description | +|---|---|---| +| `inner` | required | Any `LLMGeneratorProtocol` adapter to wrap. | +| `tracer` | auto | OTel `Tracer`. Defaults to `trace.get_tracer("learning_commons_arize_scorers")`. | +| `capture_message_content` | `False` | Set `True` to include prompt/response text in spans. Off by default — student data may be sensitive. | + +## Development + +```bash +pip install -e sdks/python +pip install -e "integrations/arize-python[dev]" +pytest integrations/arize-python/tests/ +``` diff --git a/integrations/braintrust-python/README.md b/integrations/braintrust-python/README.md new file mode 100644 index 00000000..bb562d0b --- /dev/null +++ b/integrations/braintrust-python/README.md @@ -0,0 +1,61 @@ +# learning-commons-braintrust-scorers + +[Braintrust](https://braintrust.dev/) adapters for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +Two adapters are provided: + +- **`BraintrustAnthropicAdapter`** — uses `braintrust.auto_instrument()` to intercept Anthropic SDK calls. Requires the `[braintrust]` optional dependency. +- **`BraintrustProxyAdapter`** — routes calls through the Braintrust AI Proxy. No Braintrust SDK required. + +## Installation + +```bash +# Proxy adapter only (no Braintrust SDK needed) +pip install learning-commons-braintrust-scorers + +# Auto-instrument adapter +pip install "learning-commons-braintrust-scorers[braintrust]" +``` + +## Usage + +```python +from learning_commons_braintrust_scorers import BraintrustProxyAdapter +from learning_commons_evaluators import GradeLevelAppropriatenessEvaluator +from learning_commons_evaluators.config import create_config_no_telemetry + +adapter = BraintrustProxyAdapter( + model="claude-opus-4-8-20250514", + api_key="bt-...", + project="my-project", +) +evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, +) +``` + +## Configuration + +### `BraintrustAnthropicAdapter` + +| Parameter | Default | Description | +|---|---|---| +| `model` | `"claude-opus-4-8-20250514"` | Anthropic model ID. | +| `project` | `None` | Braintrust project name. When set, calls `braintrust.init(project=...)`. | + +### `BraintrustProxyAdapter` + +| Parameter | Default | Description | +|---|---|---| +| `model` | `"claude-opus-4-8-20250514"` | Anthropic model ID. | +| `api_key` | env `BRAINTRUST_API_KEY` | Braintrust API key. Raises `ValueError` if absent. | +| `project` | `None` | Braintrust project name (passed as `x-bt-parent` header). | + +## Development + +```bash +pip install -e sdks/python +pip install -e "integrations/braintrust-python[dev]" +pytest integrations/braintrust-python/tests/ +``` diff --git a/integrations/braintrust-python/tests/test_adapter.py b/integrations/braintrust-python/tests/test_adapter.py index 259aaacb..5255ffe5 100644 --- a/integrations/braintrust-python/tests/test_adapter.py +++ b/integrations/braintrust-python/tests/test_adapter.py @@ -45,7 +45,9 @@ def _make_adapter(self, model: str = "claude-opus-4-8-20250514", project: str | mock_braintrust = MagicMock() with ( - patch("braintrust.auto_instrument", mock_braintrust.auto_instrument), + # sys.modules mock covers all braintrust attribute access, including auto_instrument. + # Do NOT use patch("braintrust.auto_instrument") here — it tries to import the real + # module before the sys.modules replacement is applied (ModuleNotFoundError). patch.dict("sys.modules", {"braintrust": mock_braintrust}), patch("anthropic.AsyncAnthropic", return_value=mock_client), ): diff --git a/integrations/inspect-python/README.md b/integrations/inspect-python/README.md index 0195d3c1..11005195 100644 --- a/integrations/inspect-python/README.md +++ b/integrations/inspect-python/README.md @@ -9,7 +9,7 @@ pip install learning-commons-inspect-scorers ``` > **Note:** Requires `learning-commons-evaluators>=0.2.0`. During local development -> (before 0.2.0 is published), install the SDK from the repo root first: +> install the SDK from the repo root first: > ```bash > pip install -e sdks/python > pip install -e integrations/inspect-python @@ -27,19 +27,13 @@ from inspect_ai import Task, task from inspect_ai.dataset import csv_dataset, FieldSpec from inspect_ai.solver import generate from learning_commons_inspect_scorers import gla_scorer -from learning_commons_evaluators.config import create_config_no_telemetry -from learning_commons_evaluators.schemas.config import GoogleLLMProviderConfig - -config = create_config_no_telemetry( - google_llm_provider_config=GoogleLLMProviderConfig(api_key="your-key"), -) @task def my_eval(): return Task( dataset=csv_dataset("samples.csv"), # requires target_grade column solver=[generate()], - scorer=gla_scorer(config=config), + scorer=gla_scorer(), ) ``` @@ -49,7 +43,7 @@ The dataset CSV must include a `target_grade` metadata column with one of: ### Scoring artifact files (edu-panda-skill-harness) ```python -scorer=gla_scorer(config=config, text_source="artifacts") +scorer=gla_scorer(text_source="artifacts") ``` ### Re-scoring an existing log from the CLI @@ -64,7 +58,7 @@ inspect score logs/my-eval.eval --scorer learning_commons_inspect_scorers/gla_sc | Parameter | Default | Description | |---|---|---| -| `config` | env vars | `EvaluatorConfig`. If `None`, reads `GOOGLE_API_KEY`, `ANTHROPIC_API_KEY`, or `OPENAI_API_KEY` from the environment. | +| `grader_model` | `"anthropic/claude-opus-4-8"` | Inspect model string for the grading LLM. Uses Inspect's model system — no separate API key configuration needed. | | `text_source` | `"completion"` | `"completion"` scores `state.output.completion`; `"artifacts"` joins `state.metadata["artifacts"]` file contents. | | `target_grade_key` | `"target_grade"` | Metadata key holding the expected grade band. | | `allow_adjacent` | `True` | If `True`, the one grade band above or below the target also passes. | diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py index 5c0f1b90..3f56cdae 100644 --- a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py @@ -16,7 +16,6 @@ APIError, ConfigurationError, InputValidationError, - OutputValidationError, ) from learning_commons_evaluators.schemas.grade_level_appropriateness import GradeLevelAnswer diff --git a/integrations/langfuse-python/README.md b/integrations/langfuse-python/README.md new file mode 100644 index 00000000..11dec6d4 --- /dev/null +++ b/integrations/langfuse-python/README.md @@ -0,0 +1,50 @@ +# learning-commons-langfuse-scorers + +[Langfuse](https://langfuse.com/) tracing adapter for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +Wraps any `LLMGeneratorProtocol` adapter and records generations in Langfuse v2. + +> **Note:** Requires `langfuse>=2.0.0,<3.0.0`. Langfuse v3+ replaced the `trace()`/`generation()` API with an OTel-based pattern — migration is tracked as a TODO. + +## Installation + +```bash +pip install learning-commons-langfuse-scorers +``` + +## Usage + +```python +from learning_commons_langfuse_scorers import LangfuseTracingAdapter +from learning_commons_inspect_scorers.adapter import InspectModelAdapter +from learning_commons_evaluators import GradeLevelAppropriatenessEvaluator +from learning_commons_evaluators.config import create_config_no_telemetry + +adapter = LangfuseTracingAdapter( + InspectModelAdapter("anthropic/claude-opus-4-8"), + trace_name="gla-eval", +) +evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, +) +``` + +> **Note:** Each `generate()` call creates a new Langfuse trace. For multi-step evaluators, +> pass a per-run unique `trace_name` (e.g. a UUID) to group calls by name in the UI. + +## Configuration + +| Parameter | Default | Description | +|---|---|---| +| `inner` | required | Any `LLMGeneratorProtocol` adapter to wrap. | +| `langfuse` | auto | `Langfuse()` client. Reads `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST` from env. | +| `trace_name` | `"lc_eval"` | Langfuse trace name. | + +## Development + +```bash +pip install -e sdks/python +pip install -e "integrations/langfuse-python[dev]" +pytest integrations/langfuse-python/tests/ +``` From e4ba334c65c7cb1816f24bc39805c21b77b55b03 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 22:11:19 -0700 Subject: [PATCH 09/10] style: remove redundant inline comments, keep non-obvious WHY comments --- .../inspect-python/src/learning_commons_inspect_scorers/gla.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py index 3f56cdae..de304925 100644 --- a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py @@ -128,7 +128,7 @@ async def score(state: TaskState, target: Target) -> Score | None: metadata={"target_grade": target_grade, "gla_grade": None}, ) - gla_grade: str = result.answer.score # e.g. "6-8" + gla_grade: str = result.answer.score passed = gla_grade in _acceptable_bands(target_grade, allow_adjacent) return Score( From 48731b9b4e00aa070470b06b948f9d9191ed23f1 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Fri, 12 Jun 2026 09:59:30 -0700 Subject: [PATCH 10/10] refactor: split Inspect integration into its own PR; keep arize/langfuse/braintrust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Inspect integration moves to ahussain/inspect-integration (its own PR) since it has a real consumer and can be validated/merged independently. This branch now carries only the speculative observability integrations (arize, langfuse, braintrust) to be revisited — likely split further by vendor — once each is validated against a real account. Also applies the high-confidence Langfuse fix: generation.end() now uses the current usage_details kwarg instead of the deprecated usage kwarg (silently dropped in recent 2.x, losing token counts in the UI). Deferred to the per-vendor revisit (need real-account validation, not doc-reading): arize span-name, braintrust invalid model ID / proxy URL / init_logger / dep bound. --- .release-please-manifest.json | 1 - integrations/inspect-python/.gitignore | 6 - integrations/inspect-python/CHANGELOG.md | 1 - integrations/inspect-python/README.md | 73 ---- integrations/inspect-python/pyproject.toml | 74 ---- .../__init__.py | 6 - .../_registry.py | 8 - .../adapter.py | 70 ---- .../learning_commons_inspect_scorers/gla.py | 150 -------- .../learning_commons_inspect_scorers/py.typed | 0 .../inspect-python/tests/test_gla_scorer.py | 341 ------------------ .../adapter.py | 4 +- .../langfuse-python/tests/test_adapter.py | 2 +- release-please-config.json | 8 - 14 files changed, 4 insertions(+), 740 deletions(-) delete mode 100644 integrations/inspect-python/.gitignore delete mode 100644 integrations/inspect-python/CHANGELOG.md delete mode 100644 integrations/inspect-python/README.md delete mode 100644 integrations/inspect-python/pyproject.toml delete mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py delete mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py delete mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py delete mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py delete mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed delete mode 100644 integrations/inspect-python/tests/test_gla_scorer.py diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 8f8d7d06..078df5d1 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -2,7 +2,6 @@ "evals/prompts": "1.5.0", "sdks/python": "0.2.0", "sdks/typescript": "0.7.0", - "integrations/inspect-python": "0.1.0", "integrations/langfuse-python": "0.1.0", "integrations/arize-python": "0.1.0", "integrations/braintrust-python": "0.1.0" diff --git a/integrations/inspect-python/.gitignore b/integrations/inspect-python/.gitignore deleted file mode 100644 index 5ca865e5..00000000 --- a/integrations/inspect-python/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.egg-info/ -dist/ -build/ -__pycache__/ -.pytest_cache/ -.mypy_cache/ diff --git a/integrations/inspect-python/CHANGELOG.md b/integrations/inspect-python/CHANGELOG.md deleted file mode 100644 index 825c32f0..00000000 --- a/integrations/inspect-python/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -# Changelog diff --git a/integrations/inspect-python/README.md b/integrations/inspect-python/README.md deleted file mode 100644 index 11005195..00000000 --- a/integrations/inspect-python/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# learning-commons-inspect-scorers - -[Inspect AI](https://inspect.aisi.org.uk/) scorer wrappers for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. - -## Installation - -```bash -pip install learning-commons-inspect-scorers -``` - -> **Note:** Requires `learning-commons-evaluators>=0.2.0`. During local development -> install the SDK from the repo root first: -> ```bash -> pip install -e sdks/python -> pip install -e integrations/inspect-python -> ``` - -## Usage - -### Grade Level Appropriateness scorer - -Evaluates whether model output (or generated artifact files) is written at the -appropriate reading level for a target K-12 grade band. - -```python -from inspect_ai import Task, task -from inspect_ai.dataset import csv_dataset, FieldSpec -from inspect_ai.solver import generate -from learning_commons_inspect_scorers import gla_scorer - -@task -def my_eval(): - return Task( - dataset=csv_dataset("samples.csv"), # requires target_grade column - solver=[generate()], - scorer=gla_scorer(), - ) -``` - -The dataset CSV must include a `target_grade` metadata column with one of: -`K-1`, `2-3`, `4-5`, `6-8`, `9-10`, `11-CCR`. - -### Scoring artifact files (edu-panda-skill-harness) - -```python -scorer=gla_scorer(text_source="artifacts") -``` - -### Re-scoring an existing log from the CLI - -Once installed, scorers are registered via Inspect's entry point system: - -```bash -inspect score logs/my-eval.eval --scorer learning_commons_inspect_scorers/gla_scorer -``` - -## Configuration - -| Parameter | Default | Description | -|---|---|---| -| `grader_model` | `"anthropic/claude-opus-4-8"` | Inspect model string for the grading LLM. Uses Inspect's model system — no separate API key configuration needed. | -| `text_source` | `"completion"` | `"completion"` scores `state.output.completion`; `"artifacts"` joins `state.metadata["artifacts"]` file contents. | -| `target_grade_key` | `"target_grade"` | Metadata key holding the expected grade band. | -| `allow_adjacent` | `True` | If `True`, the one grade band above or below the target also passes. | - -## Development - -```bash -# From repo root -pip install -e sdks/python -pip install -e "integrations/inspect-python[dev]" -pytest integrations/inspect-python/tests/ -``` diff --git a/integrations/inspect-python/pyproject.toml b/integrations/inspect-python/pyproject.toml deleted file mode 100644 index 8ec39cb4..00000000 --- a/integrations/inspect-python/pyproject.toml +++ /dev/null @@ -1,74 +0,0 @@ -[build-system] -requires = ["setuptools>=61", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "learning-commons-inspect-scorers" -version = "0.1.0" -description = "Inspect AI scorer wrappers for Learning Commons evaluators" -readme = "README.md" -license = { text = "MIT" } -requires-python = ">=3.10" -authors = [{ name = "Learning Commons" }] -keywords = ["education", "evaluators", "inspect", "evals", "scoring"] -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Topic :: Education", -] -dependencies = [ - "learning-commons-evaluators>=0.2.0", - "inspect-ai>=0.3.2", -] - -[project.optional-dependencies] -dev = [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "ruff>=0.9.0", - "mypy>=1.14.0", -] - -[project.urls] -Homepage = "https://github.com/learning-commons-org/evaluators" -Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/inspect-python" -Documentation = "https://docs.learningcommons.org/evaluators" -"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" - -# Registers scorers with Inspect's component discovery via setuptools entry points. -# Once installed, scorers are accessible as e.g. `learning_commons_inspect_scorers/gla_scorer` -# from the CLI: inspect score log.eval --scorer learning_commons_inspect_scorers/gla_scorer -[project.entry-points.inspect_ai] -learning_commons_inspect_scorers = "learning_commons_inspect_scorers._registry" - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -learning_commons_inspect_scorers = ["py.typed"] - -[tool.pytest.ini_options] -asyncio_mode = "auto" -testpaths = ["tests"] - -[tool.ruff] -target-version = "py310" -line-length = 100 - -[tool.ruff.lint] -select = ["E", "W", "F", "I", "UP", "B", "SIM"] -ignore = ["E501"] - -[tool.mypy] -python_version = "3.10" -mypy_path = ["src", "tests"] -explicit_package_bases = true -plugins = ["pydantic.mypy"] -warn_unused_configs = true -show_error_codes = true diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py deleted file mode 100644 index c9f2f977..00000000 --- a/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Learning Commons Inspect scorers — Inspect AI wrappers for LC evaluators.""" - -from learning_commons_inspect_scorers.adapter import InspectModelAdapter -from learning_commons_inspect_scorers.gla import gla_scorer - -__all__ = ["InspectModelAdapter", "gla_scorer"] diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py deleted file mode 100644 index 7e353db4..00000000 --- a/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Entry point registry — imported by Inspect via the inspect_ai setuptools entry point. - -Importing this module registers all scorers with Inspect's component system, -making them accessible by name (e.g. learning_commons_inspect_scorers/gla_scorer) -from both the Python API and the CLI. -""" - -from learning_commons_inspect_scorers.gla import gla_scorer # noqa: F401 — import triggers @scorer registry side-effect diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py deleted file mode 100644 index 448355f2..00000000 --- a/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Inspect AI model adapter implementing LLMGeneratorProtocol. - -This is a separate package (``learning-commons-inspect-scorers``) rather than -part of ``learning-commons-evaluators`` because it introduces ``inspect-ai`` as -a hard dependency — a heavy framework that not all SDK users need. - -**Versioning contract**: this package requires ``learning-commons-evaluators>=0.2.0`` -where ``LLMGeneratorProtocol`` was introduced. If a new method is added to the -protocol, bump the lower bound here and update this adapter. - -**Building a new integration** (e.g. ``integrations/langsmith-python``): -implement ``LLMGeneratorProtocol`` — a single async ``generate()`` method that -calls your framework's model and returns ``LLMResponse`` — then inject it into -any evaluator via ``GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter)``. -""" - -from __future__ import annotations - -from inspect_ai.model import ( - ChatMessageSystem, - ChatMessageUser, - GenerateConfig as InspectGenConfig, - get_model, -) - -from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse - - -class InspectModelAdapter: - """Wraps Inspect's get_model() to satisfy LLMGeneratorProtocol. - - Pass a model string in the same form accepted by Inspect's ``--model`` flag, - for example ``"anthropic/claude-opus-4-8"`` or ``"openai/gpt-4o"``. - - Example:: - - adapter = InspectModelAdapter("anthropic/claude-opus-4-8") - evaluator = GradeLevelAppropriatenessEvaluator( - config=create_config_no_telemetry(), - llm_provider=adapter, - ) - """ - - def __init__(self, model_name: str) -> None: - self._model_name = model_name - - async def generate( - self, - *, - system: str, - human: str, - config: GenerateConfig | None = None, - ) -> LLMResponse: - # get_model() is memoized by Inspect — repeated calls with the same string - # return the cached Model object without reconstruction. - inspect_model = get_model(self._model_name) - inspect_config = InspectGenConfig( - temperature=config.temperature if config is not None else None, - max_tokens=config.max_tokens if config is not None else None, - ) - output = await inspect_model.generate( - [ChatMessageSystem(content=system), ChatMessageUser(content=human)], - config=inspect_config, - ) - return LLMResponse( - content=output.completion, - model=self._model_name, - input_tokens=getattr(output.usage, "input_tokens", None), - output_tokens=getattr(output.usage, "output_tokens", None), - ) diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py deleted file mode 100644 index de304925..00000000 --- a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Inspect scorer wrapper for the Grade Level Appropriateness evaluator.""" - -from __future__ import annotations - -from pathlib import Path - -from inspect_ai.scorer import CORRECT, INCORRECT, Score, Target, accuracy, scorer -from inspect_ai.solver import TaskState - -from learning_commons_evaluators.config import create_config_no_telemetry -from learning_commons_evaluators.evaluators.grade_level_appropriateness import ( - GradeLevelAppropriatenessEvaluationInput, - GradeLevelAppropriatenessEvaluator, -) -from learning_commons_evaluators.schemas.errors import ( - APIError, - ConfigurationError, - InputValidationError, -) -from learning_commons_evaluators.schemas.grade_level_appropriateness import GradeLevelAnswer - -from learning_commons_inspect_scorers.adapter import InspectModelAdapter - -GRADE_BANDS = [m.score for m in GradeLevelAnswer] -MAX_ARTIFACT_CHARS = 20_000 -MAX_TOTAL_ARTIFACT_CHARS = 90_000 # safely under the GLA evaluator's 100k limit - - -def _completion_text(state: TaskState) -> str: - return (state.output.completion or "")[:MAX_ARTIFACT_CHARS] - - -def _artifact_text(state: TaskState) -> str: - artifacts = state.metadata.get("artifacts", []) - parts = [] - for a in artifacts: - path_str = a.get("path", "") - filename = a.get("filename", "unknown") - try: - content = Path(path_str).read_text(encoding="utf-8")[:MAX_ARTIFACT_CHARS] - parts.append(f"### {filename}\n\n{content}") - except Exception: - # Skip unreadable artifacts rather than feeding error strings to the LLM. - pass - joined = "\n\n---\n\n".join(parts) if parts else "" - return joined[:MAX_TOTAL_ARTIFACT_CHARS] - - -def _acceptable_bands(target: str, allow_adjacent: bool) -> set[str]: - idx = GRADE_BANDS.index(target) if target in GRADE_BANDS else -1 - if idx == -1 or not allow_adjacent: - return {target} - return {GRADE_BANDS[i] for i in range(max(0, idx - 1), min(len(GRADE_BANDS), idx + 2))} - - -@scorer(metrics=[accuracy()]) -def gla_scorer( - grader_model: str = "anthropic/claude-opus-4-8", - text_source: str = "completion", - target_grade_key: str = "target_grade", - allow_adjacent: bool = True, -): - """Score output for grade-level appropriateness against a target grade band. - - Uses Inspect's active model (via InspectModelAdapter) to run the GLA evaluator. - No separate LLM API keys are required — the model is resolved through Inspect's - own model configuration. - - Args: - grader_model: Inspect model string for the grading LLM. - Default: ``"anthropic/claude-opus-4-8"``. - text_source: Where to read the text to evaluate. Must be ``"completion"`` - (default, uses ``state.output.completion``) or ``"artifacts"`` - (concatenates ``state.metadata["artifacts"]`` file contents). - target_grade_key: Metadata key holding the expected grade band string. - Default: ``"target_grade"``. Must be one of: K-1, 2-3, - 4-5, 6-8, 9-10, 11-CCR. - allow_adjacent: If ``True`` (default), the one grade band above or below - the target also counts as a pass. Set to ``False`` for an - exact match. - - Returns ``Score.unscored()`` when ``target_grade_key`` is absent or not a valid - grade band, when no text is available to evaluate, or when a transient API/parse - error occurs. Re-raises ``ConfigurationError`` and ``InputValidationError`` as - task-level failures. - - ``Score.value`` is ``CORRECT`` (pass) or ``INCORRECT`` (fail). - ``Score.metadata`` contains ``gla_grade``, ``target_grade``, ``alternative_grade``, - and ``scaffolding_needed``. - """ - if text_source not in ("completion", "artifacts"): - raise ValueError(f"text_source must be 'completion' or 'artifacts', got {text_source!r}") - - adapter = InspectModelAdapter(grader_model) - # config is required by BaseEvaluator but no API keys are read here — - # the llm_provider bypasses the LangChain provider path entirely. - config = create_config_no_telemetry() - evaluator = GradeLevelAppropriatenessEvaluator(config=config, llm_provider=adapter) - get_text = _artifact_text if text_source == "artifacts" else _completion_text - - async def score(state: TaskState, target: Target) -> Score | None: - target_grade: str = (state.metadata.get(target_grade_key) or "").strip() - if not target_grade or target_grade not in GRADE_BANDS: - return Score.unscored( - explanation=( - f"{target_grade_key!r} is missing or not a valid grade band. " - f"Valid bands: {', '.join(GRADE_BANDS)}" - ), - metadata={"target_grade_key": target_grade_key, "target_grade": target_grade or None}, - ) - - text = get_text(state) - if not text.strip(): - return Score.unscored( - explanation="No text to evaluate (empty completion or no readable artifacts).", - metadata={"target_grade": target_grade, "gla_grade": None}, - ) - - try: - result = await evaluator.evaluate( - GradeLevelAppropriatenessEvaluationInput(text=text) - ) - except (ConfigurationError, InputValidationError): - raise # setup/programming errors — let Inspect surface them as task failures - except APIError as exc: # OutputValidationError is a subclass of APIError - return Score.unscored( - explanation=f"GLA evaluation failed: {exc}", - metadata={"target_grade": target_grade, "gla_grade": None}, - ) - - gla_grade: str = result.answer.score - passed = gla_grade in _acceptable_bands(target_grade, allow_adjacent) - - return Score( - value=CORRECT if passed else INCORRECT, - answer=gla_grade, - explanation=( - f"Target: {target_grade} | GLA: {gla_grade} | " - f"{'PASS' if passed else 'FAIL'}\n" - + (result.explanation.summary or "") - ), - metadata={ - "gla_grade": gla_grade, - "target_grade": target_grade, - "alternative_grade": result.explanation.details["alternative_grade"], - "scaffolding_needed": result.explanation.details["scaffolding_needed"], - }, - ) - - return score diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed b/integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed deleted file mode 100644 index e69de29b..00000000 diff --git a/integrations/inspect-python/tests/test_gla_scorer.py b/integrations/inspect-python/tests/test_gla_scorer.py deleted file mode 100644 index 5d7edd63..00000000 --- a/integrations/inspect-python/tests/test_gla_scorer.py +++ /dev/null @@ -1,341 +0,0 @@ -"""Tests for the GLA Inspect scorer wrapper.""" - -from __future__ import annotations - -import math -from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from inspect_ai.scorer import CORRECT, INCORRECT - -from learning_commons_evaluators.schemas.errors import APIError, ConfigurationError -from learning_commons_inspect_scorers.gla import _acceptable_bands, gla_scorer - -GRADE_BANDS = ["K-1", "2-3", "4-5", "6-8", "9-10", "11-CCR"] - - -# ── Helpers ────────────────────────────────────────────────────────────────── - - -def _make_state(completion: str = "", metadata: dict | None = None) -> MagicMock: - state = MagicMock() - state.output.completion = completion - state.metadata = metadata or {} - return state - - -def _make_target() -> MagicMock: - return MagicMock() - - -def _make_gla_result(grade: str = "6-8") -> MagicMock: - result = MagicMock() - result.answer.score = grade - result.explanation.summary = "Test reasoning." - result.explanation.details = { - "alternative_grade": "4-5", - "scaffolding_needed": "Pre-teach vocabulary.", - } - return result - - -def _make_scorer(grade_result: str = "6-8", side_effect=None, **scorer_kwargs): - """Build a gla_scorer with a patched evaluator. - - Patches GradeLevelAppropriatenessEvaluator so no real LLM calls are made. - The evaluator instance captured in the scorer closure is the mock, so calls - work normally after construction. - """ - mock_evaluator = MagicMock() - mock_evaluator.evaluate = AsyncMock( - return_value=_make_gla_result(grade_result), - side_effect=side_effect, - ) - with patch( - "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", - return_value=mock_evaluator, - ): - scorer_fn = gla_scorer(**scorer_kwargs) - # The scorer closure already holds the mock_evaluator instance — no further - # patching needed for subsequent calls. - return scorer_fn, mock_evaluator - - -# ── _acceptable_bands ──────────────────────────────────────────────────────── - - -class TestAcceptableBands: - def test_middle_grade_allow_adjacent(self): - assert _acceptable_bands("6-8", allow_adjacent=True) == {"4-5", "6-8", "9-10"} - - def test_lower_boundary_allow_adjacent(self): - # K-1 is at index 0 — no band below it - assert _acceptable_bands("K-1", allow_adjacent=True) == {"K-1", "2-3"} - - def test_upper_boundary_allow_adjacent(self): - # 11-CCR is at end — no band above it - assert _acceptable_bands("11-CCR", allow_adjacent=True) == {"9-10", "11-CCR"} - - def test_exact_match_only(self): - assert _acceptable_bands("6-8", allow_adjacent=False) == {"6-8"} - - def test_invalid_grade_returns_singleton(self): - assert _acceptable_bands("invalid", allow_adjacent=True) == {"invalid"} - - @pytest.mark.parametrize("band", GRADE_BANDS) - def test_all_bands_are_valid(self, band): - result = _acceptable_bands(band, allow_adjacent=True) - assert band in result - assert len(result) >= 1 - - -# ── gla_scorer — factory validation ────────────────────────────────────────── - - -class TestGlaScorerFactory: - def test_invalid_text_source_raises(self): - with pytest.raises(ValueError, match="text_source must be"): - gla_scorer(text_source="html") - - def test_valid_text_sources_accepted(self): - for src in ("completion", "artifacts"): - with patch("learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator"): - gla_scorer(text_source=src) # must not raise - - -# ── gla_scorer — score routing ──────────────────────────────────────────────── - - -class TestGlaScorer: - async def test_matching_grade_returns_correct(self): - scorer_fn, _ = _make_scorer(grade_result="6-8") - state = _make_state(completion="Sample text.", metadata={"target_grade": "6-8"}) - score = await scorer_fn(state, _make_target()) - assert score.value == CORRECT - assert score.answer == "6-8" - - async def test_adjacent_grade_passes_when_allow_adjacent(self): - scorer_fn, _ = _make_scorer(grade_result="4-5") - state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) - assert (await scorer_fn(state, _make_target())).value == CORRECT - - async def test_non_adjacent_grade_fails(self): - scorer_fn, _ = _make_scorer(grade_result="K-1") - state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) - assert (await scorer_fn(state, _make_target())).value == INCORRECT - - async def test_exact_match_only_when_adjacent_disabled(self): - scorer_fn, _ = _make_scorer(grade_result="4-5", allow_adjacent=False) - state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) - assert (await scorer_fn(state, _make_target())).value == INCORRECT - - async def test_boundary_k1_adjacent_passes(self): - scorer_fn, _ = _make_scorer(grade_result="2-3") - state = _make_state(completion="Sample.", metadata={"target_grade": "K-1"}) - assert (await scorer_fn(state, _make_target())).value == CORRECT - - async def test_boundary_11ccr_adjacent_passes(self): - scorer_fn, _ = _make_scorer(grade_result="9-10") - state = _make_state(completion="Sample.", metadata={"target_grade": "11-CCR"}) - assert (await scorer_fn(state, _make_target())).value == CORRECT - - async def test_missing_target_grade_returns_unscored(self): - scorer_fn, _ = _make_scorer() - state = _make_state(completion="Sample.", metadata={}) - score = await scorer_fn(state, _make_target()) - assert math.isnan(float(score.value)) - - async def test_invalid_target_grade_returns_unscored(self): - scorer_fn, _ = _make_scorer() - state = _make_state(completion="Sample.", metadata={"target_grade": "Grade 5"}) - score = await scorer_fn(state, _make_target()) - assert math.isnan(float(score.value)) - - async def test_empty_completion_returns_unscored(self): - scorer_fn, mock_evaluator = _make_scorer() - state = _make_state(completion="", metadata={"target_grade": "6-8"}) - score = await scorer_fn(state, _make_target()) - assert math.isnan(float(score.value)) - mock_evaluator.evaluate.assert_not_called() - - async def test_api_error_returns_unscored(self): - scorer_fn, _ = _make_scorer(side_effect=APIError("rate limit")) - state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) - score = await scorer_fn(state, _make_target()) - assert math.isnan(float(score.value)) - assert "rate limit" in score.explanation - - async def test_configuration_error_propagates(self): - scorer_fn, _ = _make_scorer(side_effect=ConfigurationError("no key")) - state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) - with pytest.raises(ConfigurationError): - await scorer_fn(state, _make_target()) - - async def test_score_metadata_populated(self): - scorer_fn, _ = _make_scorer(grade_result="6-8") - state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) - score = await scorer_fn(state, _make_target()) - assert score.metadata["gla_grade"] == "6-8" - assert score.metadata["target_grade"] == "6-8" - assert score.metadata["alternative_grade"] == "4-5" - assert score.metadata["scaffolding_needed"] == "Pre-teach vocabulary." - - async def test_custom_target_grade_key(self): - scorer_fn, _ = _make_scorer(grade_result="6-8", target_grade_key="expected_grade") - state = _make_state(completion="Sample.", metadata={"expected_grade": "6-8"}) - assert (await scorer_fn(state, _make_target())).value == CORRECT - - async def test_custom_target_grade_key_absent_returns_unscored(self): - scorer_fn, _ = _make_scorer(target_grade_key="expected_grade") - state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) - score = await scorer_fn(state, _make_target()) - assert math.isnan(float(score.value)) - - async def test_custom_grader_model_passed_to_adapter(self): - with patch( - "learning_commons_inspect_scorers.gla.InspectModelAdapter" - ) as mock_adapter_cls, patch( - "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator" - ): - gla_scorer(grader_model="openai/gpt-4o") - mock_adapter_cls.assert_called_once_with("openai/gpt-4o") - - -# ── gla_scorer — artifacts text source ──────────────────────────────────────── - - -class TestGlaScorerArtifacts: - async def test_reads_artifact_files(self, tmp_path: Path): - artifact = tmp_path / "lesson.md" - artifact.write_text("The mitochondria is the powerhouse of the cell.") - - scorer_fn, mock_evaluator = _make_scorer( - grade_result="6-8", text_source="artifacts" - ) - state = _make_state( - metadata={ - "target_grade": "6-8", - "artifacts": [{"path": str(artifact), "filename": "lesson.md"}], - } - ) - score = await scorer_fn(state, _make_target()) - assert score.value == CORRECT - called_text = mock_evaluator.evaluate.call_args[0][0].text.value - assert "mitochondria" in called_text - - async def test_unreadable_artifact_returns_unscored(self): - scorer_fn, mock_evaluator = _make_scorer(text_source="artifacts") - state = _make_state( - metadata={ - "target_grade": "6-8", - "artifacts": [{"path": "/nonexistent/path.md", "filename": "missing.md"}], - } - ) - score = await scorer_fn(state, _make_target()) - assert math.isnan(float(score.value)) - mock_evaluator.evaluate.assert_not_called() - - async def test_multiple_artifacts_joined_with_separator(self, tmp_path: Path): - f1 = tmp_path / "a.md" - f2 = tmp_path / "b.md" - f1.write_text("First document.") - f2.write_text("Second document.") - - scorer_fn, mock_evaluator = _make_scorer( - grade_result="4-5", text_source="artifacts" - ) - state = _make_state( - metadata={ - "target_grade": "4-5", - "artifacts": [ - {"path": str(f1), "filename": "a.md"}, - {"path": str(f2), "filename": "b.md"}, - ], - } - ) - await scorer_fn(state, _make_target()) - called_text = mock_evaluator.evaluate.call_args[0][0].text.value - assert "First document." in called_text - assert "Second document." in called_text - assert "---" in called_text - - -# ── Integration tests (mockllm/model) ──────────────────────────────────────── - - -class TestGlaScorerIntegration: - """End-to-end tests using Inspect's built-in mockllm/model provider. - - These tests validate that gla_scorer satisfies the Inspect Scorer protocol - and wires correctly through eval() — without making any real LLM calls. - The GLA evaluator itself is still mocked at the SDK boundary. - - .. note:: - These tests are synchronous (``def``, not ``async def``) because - ``inspect_ai.eval()`` calls ``anyio.run()`` internally to start its own - event loop. Using ``async def`` under ``pytest-asyncio asyncio_mode=auto`` - would start a second event loop, causing an anyio ``ScopeMismatch`` error. - """ - - def test_scorer_wires_through_eval(self): - from unittest.mock import AsyncMock, patch - - from inspect_ai import Task, eval - from inspect_ai.dataset import Sample - from inspect_ai.solver import generate - - mock_evaluator = MagicMock() - mock_evaluator.evaluate = AsyncMock(return_value=_make_gla_result("6-8")) - - with patch( - "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", - return_value=mock_evaluator, - ): - scorer = gla_scorer() - - task = Task( - dataset=[ - Sample( - input="Write a short paragraph for 6th graders.", - metadata={"target_grade": "6-8"}, - ) - ], - solver=[generate()], - scorer=scorer, - ) - - log = eval(task, model="mockllm/model")[0] - - assert log.status == "success" - sample = log.samples[0] - assert sample.score is not None - assert sample.score.value == CORRECT - - def test_unscored_sample_does_not_count_as_failure(self): - from unittest.mock import AsyncMock, patch - - from inspect_ai import Task, eval - from inspect_ai.dataset import Sample - from inspect_ai.solver import generate - - mock_evaluator = MagicMock() - mock_evaluator.evaluate = AsyncMock(return_value=_make_gla_result("6-8")) - - with patch( - "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", - return_value=mock_evaluator, - ): - scorer = gla_scorer() - - # Sample has no target_grade — should be unscored, not INCORRECT - task = Task( - dataset=[Sample(input="Write something.", metadata={})], - solver=[generate()], - scorer=scorer, - ) - - log = eval(task, model="mockllm/model")[0] - sample = log.samples[0] - assert sample.score is not None - assert math.isnan(float(sample.score.value)) diff --git a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py index ec191ec8..3629ea61 100644 --- a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py +++ b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py @@ -78,7 +78,9 @@ async def generate( generation.end( output=response.content, model=response.model, - usage={ + # usage_details is the current Langfuse v2 kwarg; the older `usage` kwarg + # is silently dropped in recent 2.x releases, losing token counts in the UI. + usage_details={ k: v for k, v in { "input": response.input_tokens, "output": response.output_tokens, diff --git a/integrations/langfuse-python/tests/test_adapter.py b/integrations/langfuse-python/tests/test_adapter.py index 1a852c56..cbcf2976 100644 --- a/integrations/langfuse-python/tests/test_adapter.py +++ b/integrations/langfuse-python/tests/test_adapter.py @@ -147,7 +147,7 @@ async def test_ends_generation_with_response_data(self): mock_generation.end.assert_called_once_with( output="the answer", model="claude-opus-4-8", - usage={"input": 10, "output": 20}, + usage_details={"input": 10, "output": 20}, ) async def test_ends_generation_with_error_on_exception(self): diff --git a/release-please-config.json b/release-please-config.json index 874ea81a..0714308c 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -64,14 +64,6 @@ "changelog-path": "CHANGELOG.md", "component": "sdks-typescript" }, - "integrations/inspect-python": { - "release-type": "python", - "changelog-path": "CHANGELOG.md", - "component": "integrations-inspect-python", - "release-as": "0.1.0", - "bump-minor-pre-major": true, - "bump-patch-for-minor-pre-major": true - }, "integrations/langfuse-python": { "release-type": "python", "changelog-path": "CHANGELOG.md",