diff --git a/.gitignore b/.gitignore index fb6507ee..1ee9c3bd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,15 @@ .venv/ __pycache__/ *.pyc +*.egg-info/ +dist/ +build/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Inspect AI eval logs +logs/ # Jupyter Notebook .ipynb_checkpoints/ diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 297471d8..078df5d1 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,5 +1,8 @@ { "evals/prompts": "1.5.0", "sdks/python": "0.2.0", - "sdks/typescript": "0.7.0" + "sdks/typescript": "0.7.0", + "integrations/langfuse-python": "0.1.0", + "integrations/arize-python": "0.1.0", + "integrations/braintrust-python": "0.1.0" } diff --git a/integrations/arize-python/.gitignore b/integrations/arize-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/arize-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/arize-python/CHANGELOG.md b/integrations/arize-python/CHANGELOG.md new file mode 100644 index 00000000..825c32f0 --- /dev/null +++ b/integrations/arize-python/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/integrations/arize-python/README.md b/integrations/arize-python/README.md new file mode 100644 index 00000000..e724f98e --- /dev/null +++ b/integrations/arize-python/README.md @@ -0,0 +1,45 @@ +# learning-commons-arize-scorers + +[Arize/Phoenix](https://phoenix.arize.com/) OTel tracing adapter for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +Wraps any `LLMGeneratorProtocol` adapter and emits [OpenInference](https://github.com/Arize-ai/openinference) spans compatible with Arize Phoenix and any OTel backend. + +## Installation + +```bash +pip install learning-commons-arize-scorers +``` + +## Usage + +```python +from learning_commons_arize_scorers import PhoenixTracingAdapter +from learning_commons_inspect_scorers.adapter import InspectModelAdapter +from learning_commons_evaluators import GradeLevelAppropriatenessEvaluator +from learning_commons_evaluators.config import create_config_no_telemetry + +adapter = PhoenixTracingAdapter( + InspectModelAdapter("anthropic/claude-opus-4-8"), + capture_message_content=False, # False by default — K-12 privacy +) +evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, +) +``` + +## Configuration + +| Parameter | Default | Description | +|---|---|---| +| `inner` | required | Any `LLMGeneratorProtocol` adapter to wrap. | +| `tracer` | auto | OTel `Tracer`. Defaults to `trace.get_tracer("learning_commons_arize_scorers")`. | +| `capture_message_content` | `False` | Set `True` to include prompt/response text in spans. Off by default — student data may be sensitive. | + +## Development + +```bash +pip install -e sdks/python +pip install -e "integrations/arize-python[dev]" +pytest integrations/arize-python/tests/ +``` diff --git a/integrations/arize-python/pyproject.toml b/integrations/arize-python/pyproject.toml new file mode 100644 index 00000000..163a67e4 --- /dev/null +++ b/integrations/arize-python/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-arize-scorers" +version = "0.1.0" +description = "Arize/Phoenix OTel tracing adapter for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "arize", "phoenix", "opentelemetry", "tracing"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + "opentelemetry-api>=1.0.0", +] + +[project.optional-dependencies] +dev = [ + "opentelemetry-sdk>=1.0.0", + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/arize-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_arize_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/arize-python/src/learning_commons_arize_scorers/__init__.py b/integrations/arize-python/src/learning_commons_arize_scorers/__init__.py new file mode 100644 index 00000000..7b0c3a23 --- /dev/null +++ b/integrations/arize-python/src/learning_commons_arize_scorers/__init__.py @@ -0,0 +1,5 @@ +"""Learning Commons Arize scorers — OpenInference OTel tracing adapter for LC evaluators.""" + +from learning_commons_arize_scorers.adapter import PhoenixTracingAdapter + +__all__ = ["PhoenixTracingAdapter"] diff --git a/integrations/arize-python/src/learning_commons_arize_scorers/adapter.py b/integrations/arize-python/src/learning_commons_arize_scorers/adapter.py new file mode 100644 index 00000000..b6c325d4 --- /dev/null +++ b/integrations/arize-python/src/learning_commons_arize_scorers/adapter.py @@ -0,0 +1,79 @@ +"""PhoenixTracingAdapter — decorates any LLMGeneratorProtocol with OpenInference OTel spans.""" + +from __future__ import annotations + +from opentelemetry import trace +from opentelemetry.trace import Tracer +from opentelemetry.trace.status import Status, StatusCode + +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class PhoenixTracingAdapter: + """Decorator adapter: wraps any LLMGeneratorProtocol, emits OpenInference OTel spans. + + Composes with any other adapter:: + + from learning_commons_arize_scorers import PhoenixTracingAdapter + from learning_commons_inspect_scorers.adapter import InspectModelAdapter + + adapter = PhoenixTracingAdapter(InspectModelAdapter("anthropic/claude-opus-4-8")) + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) + + Args: + inner: The underlying adapter to delegate generation to. + tracer: OTel Tracer instance. Defaults to a tracer named + ``"learning_commons_arize_scorers"``. + capture_message_content: If ``True``, writes system and human prompt text + and the model response into span attributes. Defaults to ``False``. + + .. warning:: + Enabling this may capture student-submitted text and other PII + into your observability backend. Ensure your data handling + controls (FERPA, COPPA for K-12) permit this before enabling. + """ + + def __init__( + self, + inner: LLMGeneratorProtocol, + tracer: Tracer | None = None, + *, + capture_message_content: bool = False, + ) -> None: + self._inner = inner + self._tracer = tracer or trace.get_tracer("learning_commons_arize_scorers") + self._capture_message_content = capture_message_content + + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + with self._tracer.start_as_current_span("llm.generate") as span: + span.set_attribute("openinference.span.kind", "LLM") + span.set_attribute("gen_ai.operation.name", "chat") + if self._capture_message_content: + span.set_attribute("llm.input_messages.0.message.role", "system") + span.set_attribute("llm.input_messages.0.message.content", system) + span.set_attribute("llm.input_messages.1.message.role", "user") + span.set_attribute("llm.input_messages.1.message.content", human) + try: + response = await self._inner.generate(system=system, human=human, config=config) + span.set_attribute("gen_ai.response.model", response.model) + span.set_attribute("llm.model_name", response.model) + if response.input_tokens is not None: + span.set_attribute("gen_ai.usage.input_tokens", response.input_tokens) + span.set_attribute("llm.token_count.prompt", response.input_tokens) + if response.output_tokens is not None: + span.set_attribute("gen_ai.usage.output_tokens", response.output_tokens) + span.set_attribute("llm.token_count.completion", response.output_tokens) + if self._capture_message_content: + span.set_attribute("llm.output_messages.0.message.role", "assistant") + span.set_attribute("llm.output_messages.0.message.content", response.content) + return response + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR)) + raise diff --git a/integrations/arize-python/src/learning_commons_arize_scorers/py.typed b/integrations/arize-python/src/learning_commons_arize_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/arize-python/tests/test_adapter.py b/integrations/arize-python/tests/test_adapter.py new file mode 100644 index 00000000..d9ddb279 --- /dev/null +++ b/integrations/arize-python/tests/test_adapter.py @@ -0,0 +1,177 @@ +"""Tests for PhoenixTracingAdapter using InMemorySpanExporter.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse +from learning_commons_arize_scorers import PhoenixTracingAdapter + + +# ── Fixtures ────────────────────────────────────────────────────────────────── + + +@pytest.fixture() +def exporter() -> InMemorySpanExporter: + return InMemorySpanExporter() + + +@pytest.fixture() +def tracer(exporter: InMemorySpanExporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider.get_tracer("test") + + +def _make_response( + content: str = "The answer is 42.", + model: str = "claude-test", + input_tokens: int | None = 10, + output_tokens: int | None = 5, +) -> LLMResponse: + return LLMResponse( + content=content, + model=model, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + +def _make_inner(response: LLMResponse | None = None, side_effect=None) -> AsyncMock: + mock = AsyncMock() + mock.generate = AsyncMock( + return_value=response or _make_response(), + side_effect=side_effect, + ) + return mock + + +# ── Basic span emission ─────────────────────────────────────────────────────── + + +class TestPhoenixTracingAdapterSpans: + async def test_emits_one_span_per_call(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="You are helpful.", human="What is 6×7?") + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "llm.generate" + + async def test_span_kind_attribute(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + span = exporter.get_finished_spans()[0] + assert span.attributes["openinference.span.kind"] == "LLM" + assert span.attributes["gen_ai.operation.name"] == "chat" + + async def test_input_message_attributes(self, tracer, exporter): + inner = _make_inner() + # capture_message_content=True required — off by default for K-12 privacy compliance + adapter = PhoenixTracingAdapter(inner, tracer=tracer, capture_message_content=True) + await adapter.generate(system="Be concise.", human="Hello?") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["llm.input_messages.0.message.role"] == "system" + assert attrs["llm.input_messages.0.message.content"] == "Be concise." + assert attrs["llm.input_messages.1.message.role"] == "user" + assert attrs["llm.input_messages.1.message.content"] == "Hello?" + + async def test_input_message_attributes_absent_by_default(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) # capture_message_content=False + await adapter.generate(system="Be concise.", human="Hello?") + attrs = exporter.get_finished_spans()[0].attributes + assert "llm.input_messages.0.message.content" not in attrs + assert "llm.input_messages.1.message.content" not in attrs + + async def test_output_message_attributes(self, tracer, exporter): + inner = _make_inner(_make_response(content="Hi there!")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer, capture_message_content=True) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["llm.output_messages.0.message.role"] == "assistant" + assert attrs["llm.output_messages.0.message.content"] == "Hi there!" + + async def test_model_attributes(self, tracer, exporter): + inner = _make_inner(_make_response(model="claude-opus-4")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["gen_ai.response.model"] == "claude-opus-4" + assert attrs["llm.model_name"] == "claude-opus-4" + + async def test_token_count_attributes(self, tracer, exporter): + inner = _make_inner(_make_response(input_tokens=20, output_tokens=8)) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["gen_ai.usage.input_tokens"] == 20 + assert attrs["llm.token_count.prompt"] == 20 + assert attrs["gen_ai.usage.output_tokens"] == 8 + assert attrs["llm.token_count.completion"] == 8 + + async def test_none_token_counts_omitted(self, tracer, exporter): + inner = _make_inner(_make_response(input_tokens=None, output_tokens=None)) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + await adapter.generate(system="sys", human="user") + attrs = exporter.get_finished_spans()[0].attributes + assert "gen_ai.usage.input_tokens" not in attrs + assert "gen_ai.usage.output_tokens" not in attrs + + async def test_passes_config_to_inner(self, tracer, exporter): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + cfg = GenerateConfig(temperature=0.3, max_tokens=512) + await adapter.generate(system="sys", human="user", config=cfg) + inner.generate.assert_called_once_with(system="sys", human="user", config=cfg) + + async def test_returns_inner_response(self, tracer, exporter): + response = _make_response(content="42", model="gpt-test", input_tokens=3, output_tokens=1) + inner = _make_inner(response) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + result = await adapter.generate(system="sys", human="user") + assert result is response + + +# ── Exception handling ──────────────────────────────────────────────────────── + + +class TestPhoenixTracingAdapterErrors: + async def test_exception_is_recorded_on_span(self, tracer, exporter): + inner = _make_inner(side_effect=RuntimeError("boom")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + with pytest.raises(RuntimeError, match="boom"): + await adapter.generate(system="sys", human="user") + span = exporter.get_finished_spans()[0] + events = [e.name for e in span.events] + assert "exception" in events + + async def test_exception_propagates(self, tracer, exporter): + inner = _make_inner(side_effect=ValueError("bad input")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + with pytest.raises(ValueError, match="bad input"): + await adapter.generate(system="sys", human="user") + + async def test_span_still_finished_after_exception(self, tracer, exporter): + inner = _make_inner(side_effect=RuntimeError("fail")) + adapter = PhoenixTracingAdapter(inner, tracer=tracer) + with pytest.raises(RuntimeError): + await adapter.generate(system="sys", human="user") + assert len(exporter.get_finished_spans()) == 1 + + +# ── Default tracer ──────────────────────────────────────────────────────────── + + +class TestPhoenixTracingAdapterDefaultTracer: + async def test_uses_default_tracer_when_none_provided(self): + inner = _make_inner() + adapter = PhoenixTracingAdapter(inner) + result = await adapter.generate(system="sys", human="user") + assert result.content == "The answer is 42." diff --git a/integrations/braintrust-python/.gitignore b/integrations/braintrust-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/braintrust-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/braintrust-python/CHANGELOG.md b/integrations/braintrust-python/CHANGELOG.md new file mode 100644 index 00000000..b0a24abb --- /dev/null +++ b/integrations/braintrust-python/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## [0.1.0] - 2026-06-11 + +### Features + +- Initial release: `BraintrustAnthropicAdapter` and `BraintrustProxyAdapter` diff --git a/integrations/braintrust-python/README.md b/integrations/braintrust-python/README.md new file mode 100644 index 00000000..bb562d0b --- /dev/null +++ b/integrations/braintrust-python/README.md @@ -0,0 +1,61 @@ +# learning-commons-braintrust-scorers + +[Braintrust](https://braintrust.dev/) adapters for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +Two adapters are provided: + +- **`BraintrustAnthropicAdapter`** — uses `braintrust.auto_instrument()` to intercept Anthropic SDK calls. Requires the `[braintrust]` optional dependency. +- **`BraintrustProxyAdapter`** — routes calls through the Braintrust AI Proxy. No Braintrust SDK required. + +## Installation + +```bash +# Proxy adapter only (no Braintrust SDK needed) +pip install learning-commons-braintrust-scorers + +# Auto-instrument adapter +pip install "learning-commons-braintrust-scorers[braintrust]" +``` + +## Usage + +```python +from learning_commons_braintrust_scorers import BraintrustProxyAdapter +from learning_commons_evaluators import GradeLevelAppropriatenessEvaluator +from learning_commons_evaluators.config import create_config_no_telemetry + +adapter = BraintrustProxyAdapter( + model="claude-opus-4-8-20250514", + api_key="bt-...", + project="my-project", +) +evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, +) +``` + +## Configuration + +### `BraintrustAnthropicAdapter` + +| Parameter | Default | Description | +|---|---|---| +| `model` | `"claude-opus-4-8-20250514"` | Anthropic model ID. | +| `project` | `None` | Braintrust project name. When set, calls `braintrust.init(project=...)`. | + +### `BraintrustProxyAdapter` + +| Parameter | Default | Description | +|---|---|---| +| `model` | `"claude-opus-4-8-20250514"` | Anthropic model ID. | +| `api_key` | env `BRAINTRUST_API_KEY` | Braintrust API key. Raises `ValueError` if absent. | +| `project` | `None` | Braintrust project name (passed as `x-bt-parent` header). | + +## Development + +```bash +pip install -e sdks/python +pip install -e "integrations/braintrust-python[dev]" +pytest integrations/braintrust-python/tests/ +``` diff --git a/integrations/braintrust-python/pyproject.toml b/integrations/braintrust-python/pyproject.toml new file mode 100644 index 00000000..cd699b20 --- /dev/null +++ b/integrations/braintrust-python/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-braintrust-scorers" +version = "0.1.0" +description = "Braintrust adapter for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "braintrust", "evals", "scoring"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + "anthropic>=0.40.0", +] + +[project.optional-dependencies] +braintrust = [ + "braintrust>=0.0.100", +] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/braintrust-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_braintrust_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/braintrust-python/src/learning_commons_braintrust_scorers/__init__.py b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/__init__.py new file mode 100644 index 00000000..5a9664b1 --- /dev/null +++ b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/__init__.py @@ -0,0 +1,28 @@ +"""learning-commons-braintrust-scorers + +Braintrust adapters for Learning Commons evaluators. + +Adapters implement LLMGeneratorProtocol and can be passed directly to any +evaluator that accepts an ``llm_provider`` argument:: + + from learning_commons_braintrust_scorers import BraintrustAnthropicAdapter + from learning_commons_evaluators.evaluators.gla import ( + GradeLevelAppropriatenessEvaluator, + GradeLevelAppropriatenessEvaluationInput, + ) + + evaluator = GradeLevelAppropriatenessEvaluator( + config=..., + llm_provider=BraintrustAnthropicAdapter(project="my-project"), + ) + result = await evaluator.evaluate(GradeLevelAppropriatenessEvaluationInput(text="...")) + print(result.answer.score) # e.g. "6-8" + print(result.explanation.summary) # reasoning text +""" + +from learning_commons_braintrust_scorers.adapter import ( + BraintrustAnthropicAdapter, + BraintrustProxyAdapter, +) + +__all__ = ["BraintrustAnthropicAdapter", "BraintrustProxyAdapter"] diff --git a/integrations/braintrust-python/src/learning_commons_braintrust_scorers/adapter.py b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/adapter.py new file mode 100644 index 00000000..7b4f87cc --- /dev/null +++ b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/adapter.py @@ -0,0 +1,155 @@ +"""Braintrust adapters implementing LLMGeneratorProtocol. + +Two adapters share a common base that handles the Anthropic generation call: + +``BraintrustAnthropicAdapter`` + Uses ``braintrust.auto_instrument()`` to intercept Anthropic SDK calls. + Requires the ``[braintrust]`` optional dependency:: + + pip install learning-commons-braintrust-scorers[braintrust] + + Usage:: + + from learning_commons_braintrust_scorers import BraintrustAnthropicAdapter + + adapter = BraintrustAnthropicAdapter(project="my-project") + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) + +``BraintrustProxyAdapter`` + Routes calls through ``https://api.braintrust.dev/v1/proxy``. + No Braintrust SDK required — only the ``anthropic`` package:: + + pip install learning-commons-braintrust-scorers + + Usage:: + + from learning_commons_braintrust_scorers import BraintrustProxyAdapter + + adapter = BraintrustProxyAdapter(project="my-project") + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) +""" + +from __future__ import annotations + +import anthropic +from anthropic import NOT_GIVEN + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse + +_DEFAULT_MODEL = "claude-opus-4-8-20250514" +_DEFAULT_MAX_TOKENS = 4096 + + +class _AnthropicAdapterBase: + """Shared Anthropic generation logic for Braintrust adapters. + + Subclasses set ``self._client`` and ``self._model`` in ``__init__``. + """ + + _client: anthropic.AsyncAnthropic + _model: str + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + msg = await self._client.messages.create( + model=self._model, + system=system, + messages=[{"role": "user", "content": human}], + max_tokens=config.max_tokens if (config and config.max_tokens is not None) else _DEFAULT_MAX_TOKENS, + # Use NOT_GIVEN (not None) so the field is omitted from the request body. + # Passing None serialises as {"temperature": null} which the Anthropic API rejects. + temperature=config.temperature if (config and config.temperature is not None) else NOT_GIVEN, + ) + # Find the first text block; content may include ThinkingBlock or ToolUseBlock. + text_block = next((b for b in msg.content if b.type == "text"), None) + if text_block is None: + raise ValueError( + f"Anthropic response from {msg.model} contained no text block " + f"(content types: {[b.type for b in msg.content]})" + ) + return LLMResponse( + content=text_block.text, + model=msg.model, + input_tokens=msg.usage.input_tokens, + output_tokens=msg.usage.output_tokens, + ) + + async def aclose(self) -> None: + await self._client.close() + + +class BraintrustAnthropicAdapter(_AnthropicAdapterBase): + """Adapter using Braintrust auto-instrumentation of the Anthropic SDK. + + Calls ``braintrust.auto_instrument()`` at construction time (idempotent). + When ``project`` is provided, calls ``braintrust.init(project=project)`` + so that traces are associated with the correct Braintrust project. + + Requires ``pip install learning-commons-braintrust-scorers[braintrust]``. + + Args: + model: Anthropic model ID. + project: Braintrust project name. When provided, initialises the + Braintrust SDK so traces appear under this project in the UI. + """ + + def __init__( + self, + model: str = _DEFAULT_MODEL, + *, + project: str | None = None, + ) -> None: + import braintrust + + braintrust.auto_instrument() + if project: + braintrust.init(project=project) + self._client = anthropic.AsyncAnthropic() + self._model = model + + +class BraintrustProxyAdapter(_AnthropicAdapterBase): + """Adapter using Braintrust AI Proxy — no Braintrust SDK required. + + Routes Anthropic API calls through ``https://api.braintrust.dev/v1/proxy``. + Braintrust logs all calls automatically via HTTP interception. + + Args: + model: Anthropic model ID. + api_key: Braintrust API key. Falls back to the ``BRAINTRUST_API_KEY`` + environment variable. Raises ``ValueError`` if neither is set + or if the resolved value is blank. + project: Braintrust project name. Passed as the ``x-bt-parent`` header + so traces appear under the correct project in the Braintrust UI. + + Raises: + ValueError: At construction time when no non-blank API key is available. + """ + + def __init__( + self, + model: str = _DEFAULT_MODEL, + *, + api_key: str | None = None, + project: str | None = None, + ) -> None: + import os + + resolved_key = (api_key or os.environ.get("BRAINTRUST_API_KEY") or "").strip() + if not resolved_key: + raise ValueError( + "Braintrust API key is required. Provide it via the api_key argument " + "or set the BRAINTRUST_API_KEY environment variable." + ) + + self._client = anthropic.AsyncAnthropic( + base_url="https://api.braintrust.dev/v1/proxy", + auth_token=resolved_key, + default_headers={"x-bt-parent": f"project_name:{project}"} if project else {}, + ) + self._model = model diff --git a/integrations/braintrust-python/src/learning_commons_braintrust_scorers/py.typed b/integrations/braintrust-python/src/learning_commons_braintrust_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/braintrust-python/tests/test_adapter.py b/integrations/braintrust-python/tests/test_adapter.py new file mode 100644 index 00000000..5255ffe5 --- /dev/null +++ b/integrations/braintrust-python/tests/test_adapter.py @@ -0,0 +1,298 @@ +"""Tests for BraintrustAnthropicAdapter and BraintrustProxyAdapter.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _make_anthropic_message( + text: str = "response text", + model: str = "claude-opus-4-8-20250514", + input_tokens: int = 10, + output_tokens: int = 20, +) -> MagicMock: + msg = MagicMock() + text_block = MagicMock(type="text", text=text) + msg.content = [text_block] + msg.model = model + msg.usage.input_tokens = input_tokens + msg.usage.output_tokens = output_tokens + return msg + + +def _make_async_anthropic_client(message: MagicMock | None = None) -> MagicMock: + """Return a mock AsyncAnthropic client whose messages.create returns *message*.""" + client = MagicMock() + client.messages.create = AsyncMock(return_value=message or _make_anthropic_message()) + client.close = AsyncMock() + return client + + +# ── BraintrustAnthropicAdapter ──────────────────────────────────────────────── + + +class TestBraintrustAnthropicAdapter: + def _make_adapter(self, model: str = "claude-opus-4-8-20250514", project: str | None = None): + """Construct adapter with braintrust and anthropic mocked out.""" + mock_client = _make_async_anthropic_client() + mock_braintrust = MagicMock() + + with ( + # sys.modules mock covers all braintrust attribute access, including auto_instrument. + # Do NOT use patch("braintrust.auto_instrument") here — it tries to import the real + # module before the sys.modules replacement is applied (ModuleNotFoundError). + patch.dict("sys.modules", {"braintrust": mock_braintrust}), + patch("anthropic.AsyncAnthropic", return_value=mock_client), + ): + from learning_commons_braintrust_scorers.adapter import BraintrustAnthropicAdapter + + adapter = BraintrustAnthropicAdapter(model=model, project=project) + + # Attach the mock client so tests can make assertions on it. + adapter._client = mock_client + return adapter, mock_client, mock_braintrust + + def test_auto_instrument_called_at_construction(self): + mock_client = _make_async_anthropic_client() + mock_braintrust = MagicMock() + + with ( + patch.dict("sys.modules", {"braintrust": mock_braintrust}), + patch("anthropic.AsyncAnthropic", return_value=mock_client), + ): + from importlib import reload + + import learning_commons_braintrust_scorers.adapter as mod + + reload(mod) + mod.BraintrustAnthropicAdapter() + + # Use .called (not assert_called_once) — reload() inside _make_adapter() may have + # already triggered a call, making assert_called_once() order-dependent across tests. + assert mock_braintrust.auto_instrument.called + + async def test_generate_returns_llm_response(self): + msg = _make_anthropic_message( + text="some output", model="claude-opus-4-8-20250514", input_tokens=5, output_tokens=15 + ) + adapter, mock_client, _ = self._make_adapter() + mock_client.messages.create.return_value = msg + + result = await adapter.generate(system="sys prompt", human="user prompt") + + assert isinstance(result, LLMResponse) + assert result.content == "some output" + assert result.model == "claude-opus-4-8-20250514" + assert result.input_tokens == 5 + assert result.output_tokens == 15 + + async def test_generate_passes_system_and_human(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="the system", human="the human") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["system"] == "the system" + assert call_kwargs["messages"] == [{"role": "user", "content": "the human"}] + + async def test_generate_default_max_tokens(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["max_tokens"] == 4096 + + async def test_generate_default_temperature(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + call_kwargs = mock_client.messages.create.call_args[1] + # When no config is provided, temperature=NOT_GIVEN (field omitted from request) + from anthropic import NOT_GIVEN + assert call_kwargs["temperature"] is NOT_GIVEN + + async def test_generate_respects_config_max_tokens(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=None, max_tokens=512) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["max_tokens"] == 512 + + async def test_generate_respects_config_temperature(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=0.7, max_tokens=None) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["temperature"] == 0.7 + + async def test_generate_config_max_tokens_none_falls_back_to_4096(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=0.5, max_tokens=None) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["max_tokens"] == 4096 + + async def test_generate_uses_configured_model(self): + adapter, mock_client, _ = self._make_adapter(model="claude-haiku-3-5-20251022") + + await adapter.generate(system="s", human="h") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["model"] == "claude-haiku-3-5-20251022" + + async def test_aclose_calls_client_close(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.aclose() + + mock_client.close.assert_called_once() + + +# ── BraintrustProxyAdapter ──────────────────────────────────────────────────── + + +class TestBraintrustProxyAdapter: + def _make_adapter( + self, + model: str = "claude-opus-4-8-20250514", + api_key: str = "test-key", # always provide a key; test ValueError separately + project: str | None = None, + env: dict | None = None, + ): + mock_client = _make_async_anthropic_client() + captured: dict = {} + + def capture_constructor(**kwargs): + captured.update(kwargs) + return mock_client + + extra_env = env or {} + with ( + patch("anthropic.AsyncAnthropic", side_effect=capture_constructor), + patch.dict("os.environ", extra_env, clear=False), + ): + from importlib import reload + + import learning_commons_braintrust_scorers.adapter as mod + + reload(mod) + adapter = mod.BraintrustProxyAdapter(model=model, api_key=api_key, project=project) + + adapter._client = mock_client + return adapter, mock_client, captured + + def test_raises_when_no_api_key(self): + import pytest + from learning_commons_braintrust_scorers.adapter import BraintrustProxyAdapter + with patch.dict("os.environ", {}, clear=True): + with pytest.raises(ValueError, match="API key"): + BraintrustProxyAdapter(api_key=None) + + def test_proxy_base_url(self): + _, _, captured = self._make_adapter() + assert captured["base_url"] == "https://api.braintrust.dev/v1/proxy" + + def test_api_key_from_argument(self): + _, _, captured = self._make_adapter(api_key="my-key") + assert captured["auth_token"] == "my-key" + + def test_api_key_from_env(self): + # Pass api_key=None explicitly so env var is the only source + _, _, captured = self._make_adapter(api_key=None, env={"BRAINTRUST_API_KEY": "env-key"}) + assert captured["auth_token"] == "env-key" + + def test_api_key_argument_takes_precedence_over_env(self): + _, _, captured = self._make_adapter( + api_key="arg-key", env={"BRAINTRUST_API_KEY": "env-key"} + ) + assert captured["auth_token"] == "arg-key" + + def test_project_sets_bt_parent_header(self): + _, _, captured = self._make_adapter(project="my-project") + assert captured["default_headers"] == {"x-bt-parent": "project_name:my-project"} + + def test_no_project_omits_default_headers(self): + _, _, captured = self._make_adapter(project=None) + assert captured.get("default_headers", {}) == {} + + async def test_generate_returns_llm_response(self): + msg = _make_anthropic_message( + text="proxy output", model="claude-opus-4-8-20250514", input_tokens=8, output_tokens=12 + ) + adapter, mock_client, _ = self._make_adapter() + mock_client.messages.create.return_value = msg + + result = await adapter.generate(system="sys", human="usr") + + assert isinstance(result, LLMResponse) + assert result.content == "proxy output" + assert result.input_tokens == 8 + assert result.output_tokens == 12 + + async def test_generate_passes_system_and_human(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="the system", human="the human") + + call_kwargs = mock_client.messages.create.call_args[1] + assert call_kwargs["system"] == "the system" + assert call_kwargs["messages"] == [{"role": "user", "content": "the human"}] + + async def test_generate_default_max_tokens(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + assert mock_client.messages.create.call_args[1]["max_tokens"] == 4096 + + async def test_generate_default_temperature(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.generate(system="s", human="h") + + from anthropic import NOT_GIVEN + assert mock_client.messages.create.call_args[1]["temperature"] is NOT_GIVEN + + async def test_generate_respects_config(self): + adapter, mock_client, _ = self._make_adapter() + config = GenerateConfig(temperature=0.1, max_tokens=256) + + await adapter.generate(system="s", human="h", config=config) + + kwargs = mock_client.messages.create.call_args[1] + assert kwargs["max_tokens"] == 256 + assert kwargs["temperature"] == 0.1 + + async def test_generate_uses_configured_model(self): + adapter, mock_client, _ = self._make_adapter(model="claude-haiku-3-5-20251022") + + await adapter.generate(system="s", human="h") + + assert mock_client.messages.create.call_args[1]["model"] == "claude-haiku-3-5-20251022" + + async def test_aclose_calls_client_close(self): + adapter, mock_client, _ = self._make_adapter() + + await adapter.aclose() + + mock_client.close.assert_called_once() + + +# ── Protocol conformance ────────────────────────────────────────────────────── + + diff --git a/integrations/langfuse-python/.gitignore b/integrations/langfuse-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/langfuse-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/langfuse-python/CHANGELOG.md b/integrations/langfuse-python/CHANGELOG.md new file mode 100644 index 00000000..825c32f0 --- /dev/null +++ b/integrations/langfuse-python/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/integrations/langfuse-python/README.md b/integrations/langfuse-python/README.md new file mode 100644 index 00000000..11dec6d4 --- /dev/null +++ b/integrations/langfuse-python/README.md @@ -0,0 +1,50 @@ +# learning-commons-langfuse-scorers + +[Langfuse](https://langfuse.com/) tracing adapter for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +Wraps any `LLMGeneratorProtocol` adapter and records generations in Langfuse v2. + +> **Note:** Requires `langfuse>=2.0.0,<3.0.0`. Langfuse v3+ replaced the `trace()`/`generation()` API with an OTel-based pattern — migration is tracked as a TODO. + +## Installation + +```bash +pip install learning-commons-langfuse-scorers +``` + +## Usage + +```python +from learning_commons_langfuse_scorers import LangfuseTracingAdapter +from learning_commons_inspect_scorers.adapter import InspectModelAdapter +from learning_commons_evaluators import GradeLevelAppropriatenessEvaluator +from learning_commons_evaluators.config import create_config_no_telemetry + +adapter = LangfuseTracingAdapter( + InspectModelAdapter("anthropic/claude-opus-4-8"), + trace_name="gla-eval", +) +evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, +) +``` + +> **Note:** Each `generate()` call creates a new Langfuse trace. For multi-step evaluators, +> pass a per-run unique `trace_name` (e.g. a UUID) to group calls by name in the UI. + +## Configuration + +| Parameter | Default | Description | +|---|---|---| +| `inner` | required | Any `LLMGeneratorProtocol` adapter to wrap. | +| `langfuse` | auto | `Langfuse()` client. Reads `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST` from env. | +| `trace_name` | `"lc_eval"` | Langfuse trace name. | + +## Development + +```bash +pip install -e sdks/python +pip install -e "integrations/langfuse-python[dev]" +pytest integrations/langfuse-python/tests/ +``` diff --git a/integrations/langfuse-python/pyproject.toml b/integrations/langfuse-python/pyproject.toml new file mode 100644 index 00000000..68cc72e8 --- /dev/null +++ b/integrations/langfuse-python/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-langfuse-scorers" +version = "0.1.0" +description = "Langfuse tracing adapter for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "langfuse", "tracing", "observability"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + # Langfuse v3+ (released 2025) removed trace()/generation() in favour of an + # OTel-based API. Pin to v2 until this adapter is migrated to start_as_current_generation(). + # TODO: migrate to v3+ OTel pattern and remove the upper bound. + "langfuse>=2.0.0,<3.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/langfuse-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_langfuse_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/__init__.py b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/__init__.py new file mode 100644 index 00000000..6e9fa653 --- /dev/null +++ b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/__init__.py @@ -0,0 +1,5 @@ +"""Learning Commons Langfuse scorers — Langfuse tracing adapter for LC evaluators.""" + +from learning_commons_langfuse_scorers.adapter import LangfuseTracingAdapter + +__all__ = ["LangfuseTracingAdapter"] diff --git a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py new file mode 100644 index 00000000..3629ea61 --- /dev/null +++ b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/adapter.py @@ -0,0 +1,98 @@ +"""LangfuseTracingAdapter — decorator that wraps any LLMGeneratorProtocol and records Langfuse generations. + +.. note:: + This adapter targets the Langfuse v2 SDK (``langfuse>=2.0.0,<3.0.0``). + Langfuse v3+ replaced ``trace()``/``generation()`` with an OTel-based API + (``start_as_current_generation()``). A migration is tracked as a TODO. + +Usage:: + + from learning_commons_langfuse_scorers import LangfuseTracingAdapter + from learning_commons_inspect_scorers.adapter import InspectModelAdapter + + adapter = LangfuseTracingAdapter(InspectModelAdapter("anthropic/claude-opus-4-8")) + evaluator = GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter) +""" + +from __future__ import annotations + +import asyncio + +from langfuse import Langfuse + +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class LangfuseTracingAdapter: + """Decorator adapter: wraps any LLMGeneratorProtocol, records Langfuse generations. + + Args: + inner: The underlying adapter to delegate generation to. + langfuse: Langfuse client instance. Defaults to ``Langfuse()``, which + reads ``LANGFUSE_PUBLIC_KEY``, ``LANGFUSE_SECRET_KEY``, and + ``LANGFUSE_HOST`` from the environment. + trace_name: Name for the Langfuse trace. Default: ``"lc_eval"``. + + .. note:: + Each call to ``generate()`` creates a new Langfuse trace. For single-step + evaluators (GLA, conventionality) this produces one trace per evaluation. + For multi-step evaluators (vocabulary: 2 steps), this produces one trace + per LLM call — the steps appear as separate traces rather than nested + generations on a single trace. Pass a unique ``trace_name`` per evaluation + run (e.g. a UUID) if you want to group them by name in the Langfuse UI. + """ + + def __init__( + self, + inner: LLMGeneratorProtocol, + langfuse: Langfuse | None = None, + trace_name: str = "lc_eval", + ) -> None: + self._inner = inner + self._langfuse = langfuse or Langfuse() + self._trace_name = trace_name + + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + lf_trace = self._langfuse.trace(name=self._trace_name) + generation = lf_trace.generation( + name="llm_generate", + input=[ + {"role": "system", "content": system}, + {"role": "user", "content": human}, + ], + model_parameters={ + k: v for k, v in { + "temperature": config.temperature if config else None, + "max_tokens": config.max_tokens if config else None, + }.items() if v is not None + }, + ) + try: + response = await self._inner.generate(system=system, human=human, config=config) + generation.end( + output=response.content, + model=response.model, + # usage_details is the current Langfuse v2 kwarg; the older `usage` kwarg + # is silently dropped in recent 2.x releases, losing token counts in the UI. + usage_details={ + k: v for k, v in { + "input": response.input_tokens, + "output": response.output_tokens, + }.items() if v is not None + }, + ) + return response + except Exception as exc: + generation.end(level="ERROR", status_message=str(exc)) + raise + + async def aclose(self) -> None: + """Flush buffered Langfuse events. Offloads the blocking flush to a thread pool.""" + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, self._langfuse.flush) diff --git a/integrations/langfuse-python/src/learning_commons_langfuse_scorers/py.typed b/integrations/langfuse-python/src/learning_commons_langfuse_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/langfuse-python/tests/test_adapter.py b/integrations/langfuse-python/tests/test_adapter.py new file mode 100644 index 00000000..cbcf2976 --- /dev/null +++ b/integrations/langfuse-python/tests/test_adapter.py @@ -0,0 +1,181 @@ +"""Tests for LangfuseTracingAdapter.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse +from learning_commons_langfuse_scorers import LangfuseTracingAdapter + + +def _make_mock_langfuse() -> MagicMock: + """Return a Langfuse mock with the trace/generation chain wired up.""" + mock_generation = MagicMock() + mock_trace = MagicMock() + mock_trace.generation.return_value = mock_generation + mock_langfuse = MagicMock() + mock_langfuse.trace.return_value = mock_trace + return mock_langfuse + + +def _make_inner(response: LLMResponse | None = None, side_effect=None) -> MagicMock: + if response is None: + response = LLMResponse( + content='{"score": "6-8"}', + model="claude-opus-4-8", + input_tokens=10, + output_tokens=20, + ) + inner = MagicMock() + inner.generate = AsyncMock(return_value=response, side_effect=side_effect) + return inner + + +class TestLangfuseTracingAdapterInit: + def test_creates_langfuse_when_not_provided(self): + inner = _make_inner() + with patch("learning_commons_langfuse_scorers.adapter.Langfuse") as mock_cls: + mock_cls.return_value = MagicMock() + adapter = LangfuseTracingAdapter(inner) + mock_cls.assert_called_once_with() + assert adapter._inner is inner + + def test_uses_provided_langfuse_instance(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + assert adapter._langfuse is mock_langfuse + + def test_default_trace_name(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + assert adapter._trace_name == "lc_eval" + + def test_custom_trace_name(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse, trace_name="my_eval") + assert adapter._trace_name == "my_eval" + + +class TestLangfuseTracingAdapterGenerate: + async def test_creates_trace_with_configured_name(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse, trace_name="my_trace") + + await adapter.generate(system="You are a grader.", human="Assess this text.") + + mock_langfuse.trace.assert_called_once_with(name="my_trace") + + async def test_creates_generation_with_correct_input(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + mock_trace = mock_langfuse.trace.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.generate(system="sys prompt", human="user prompt") + + mock_trace.generation.assert_called_once() + call_kwargs = mock_trace.generation.call_args[1] + assert call_kwargs["name"] == "llm_generate" + assert call_kwargs["input"] == [ + {"role": "system", "content": "sys prompt"}, + {"role": "user", "content": "user prompt"}, + ] + + async def test_passes_config_model_parameters(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + mock_trace = mock_langfuse.trace.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + config = GenerateConfig(temperature=0.7, max_tokens=256) + + await adapter.generate(system="s", human="h", config=config) + + call_kwargs = mock_trace.generation.call_args[1] + assert call_kwargs["model_parameters"] == {"temperature": 0.7, "max_tokens": 256} + + async def test_none_config_sends_none_parameters(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + mock_trace = mock_langfuse.trace.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.generate(system="s", human="h", config=None) + + call_kwargs = mock_trace.generation.call_args[1] + # None values are filtered out — model_parameters is empty when config is None + assert call_kwargs["model_parameters"] == {} + + async def test_calls_inner_generate_with_correct_args(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + config = GenerateConfig(temperature=0.0, max_tokens=128) + + await adapter.generate(system="sys", human="usr", config=config) + + inner.generate.assert_called_once_with(system="sys", human="usr", config=config) + + async def test_returns_inner_response(self): + response = LLMResponse( + content="result", model="gpt-4", input_tokens=5, output_tokens=15 + ) + inner = _make_inner(response=response) + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + result = await adapter.generate(system="s", human="h") + + assert result is response + + async def test_ends_generation_with_response_data(self): + response = LLMResponse( + content="the answer", model="claude-opus-4-8", input_tokens=10, output_tokens=20 + ) + inner = _make_inner(response=response) + mock_langfuse = _make_mock_langfuse() + mock_generation = mock_langfuse.trace.return_value.generation.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.generate(system="s", human="h") + + mock_generation.end.assert_called_once_with( + output="the answer", + model="claude-opus-4-8", + usage_details={"input": 10, "output": 20}, + ) + + async def test_ends_generation_with_error_on_exception(self): + inner = _make_inner(side_effect=RuntimeError("timeout")) + mock_langfuse = _make_mock_langfuse() + mock_generation = mock_langfuse.trace.return_value.generation.return_value + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + with pytest.raises(RuntimeError, match="timeout"): + await adapter.generate(system="s", human="h") + + mock_generation.end.assert_called_once_with(level="ERROR", status_message="timeout") + + async def test_re_raises_exception_after_recording(self): + inner = _make_inner(side_effect=ValueError("bad input")) + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + with pytest.raises(ValueError, match="bad input"): + await adapter.generate(system="s", human="h") + + +class TestLangfuseTracingAdapterAclose: + async def test_aclose_flushes_langfuse(self): + inner = _make_inner() + mock_langfuse = _make_mock_langfuse() + adapter = LangfuseTracingAdapter(inner, langfuse=mock_langfuse) + + await adapter.aclose() + + mock_langfuse.flush.assert_called_once_with() diff --git a/release-please-config.json b/release-please-config.json index a0cd7202..0714308c 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -63,6 +63,30 @@ "release-type": "node", "changelog-path": "CHANGELOG.md", "component": "sdks-typescript" + }, + "integrations/langfuse-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-langfuse-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true + }, + "integrations/arize-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-arize-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true + }, + "integrations/braintrust-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-braintrust-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true } } } diff --git a/sdks/python/src/learning_commons_evaluators/__init__.py b/sdks/python/src/learning_commons_evaluators/__init__.py index 6169df43..2ec1b989 100644 --- a/sdks/python/src/learning_commons_evaluators/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/__init__.py @@ -59,6 +59,11 @@ TextInputField, ) from learning_commons_evaluators.schemas.config import EvaluationSettings, LLMProvider +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.conventionality import ( ConventionalityEvaluationSettings, ConventionalityOutput, @@ -163,6 +168,9 @@ "create_config_telemetry_with_full_input", "create_logger", "create_silent_logger", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "get_logger", "wrap_provider_error", ] diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 9954c987..70d2b412 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import json as _json +import re import time from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable @@ -32,6 +34,11 @@ EvaluationInput, EvaluationResult, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -50,6 +57,75 @@ ParsedT = TypeVar("ParsedT", bound=BaseModel) +def _parse_json_output( + raw: str, + parser_output_type: type[ParsedT], + json_dict_normalizer: Callable[[dict], dict] | None, + provider_type: Any, + model: str, +) -> ParsedT: + """Parse a raw JSON string into ``parser_output_type``, wrapping errors consistently. + + Shared by both the protocol path and (for the normalizer branch) the LangChain path + so that error handling is symmetric and normaliser logic lives in one place. + """ + raw = _strip_json_fences(raw) + try: + if json_dict_normalizer is not None: + parsed_dict = _json.loads(raw) + if not isinstance(parsed_dict, dict): + raise OutputValidationError( + "Model output is not a JSON object", + provider=provider_type, + model=model, + ) + try: + normalized = json_dict_normalizer(parsed_dict) + except (TypeError, ValueError) as norm_err: + raise OutputValidationError( + "Model output could not be normalized before validation", + provider=provider_type, + model=model, + ) from norm_err + return parser_output_type.model_validate(normalized) + return parser_output_type.model_validate_json(raw) + except PydanticValidationError as e: + raise OutputValidationError( + provider=provider_type, + model=model, + validation_errors=sanitize_pydantic_errors(e.errors()), + ) from e + except OutputValidationError: + raise + except Exception as e: + raise OutputValidationError(provider=provider_type, model=model) from e + + +def _strip_json_fences(text: str) -> str: + """Strip markdown code fences and extract the first valid JSON object or array. + + Uses ``json.JSONDecoder.raw_decode`` to locate the first balanced JSON structure + and discard any surrounding prose or trailing text. This correctly handles: + - Markdown-fenced responses: ````json\\n{...}\\n``` `` + - Prose-prefixed responses: ``Here is the result:\\n{...}`` + - Trailing-prose responses: ``{...} Here is my reasoning...`` + """ + text = text.strip() + text = re.sub(r"^```(?:json)?\s*\n?", "", text) + text = re.sub(r"\n?```\s*$", "", text) + text = text.strip() + # Find the first { or [ and use raw_decode to extract the complete JSON structure, + # correctly discarding any trailing prose or a second JSON object in the response. + start = next((i for i, ch in enumerate(text) if ch in ("{", "[")), -1) + if start != -1: + try: + _, end = _json.JSONDecoder().raw_decode(text, start) + return text[start:end] + except _json.JSONDecodeError: + pass + return text + + class BaseEvaluator(ABC, Generic[InputT, OutputT, SettingsT]): """ Abstract base class for all evaluators. @@ -74,9 +150,11 @@ def __init__( self, config: EvaluatorConfig, *, + llm_provider: LLMGeneratorProtocol | None = None, default_evaluation_settings: SettingsT | None = None, ) -> None: self.config = config + self._llm_provider = llm_provider if default_evaluation_settings is not None: self.default_evaluation_settings = default_evaluation_settings # TODO: validate config @@ -312,6 +390,14 @@ async def execute_prompt_chain_step( Parsed instance of ``parser_output_type`` when it is a model class; plain ``str`` when ``parser_output_type`` is omitted or ``None``. + Note: + **Execution path**: when ``self._llm_provider`` is set (injected at + construction via ``BaseEvaluator.__init__``), the *protocol path* is taken + — the LangChain template is formatted to extract system/human strings, the + injected provider is called directly, and JSON is parsed via Pydantic. + When ``self._llm_provider`` is ``None`` (default), the *LangChain path* + is taken and ``create_provider()`` is called internally. + Raises: ConfigurationError: No provider config for ``prompt_settings.provider_type``. OutputValidationError: The LLM response didn't satisfy the expected @@ -330,6 +416,91 @@ async def execute_prompt_chain_step( # Populated after a successful LLM invoke so we can attach usage even if parsing fails. token_usage: TokenUsage | None = None + if self._llm_provider is not None: + # ── Protocol path ───────────────────────────────────────────── + provider = self._llm_provider + + async def _run_via_provider() -> BaseModel | str: + nonlocal token_usage + try: + # Inside try: missing template variables become EvaluatorErrors, + # not bare KeyErrors — consistent with the LangChain path's error contract. + formatted = await template.aformat_messages(**chain_inputs) + system_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), + "", + ) + human_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), + "", + ) + if not human_str: + raise ValueError( + f"Template for step '{step_name}' produced no human message. " + 'Ensure the template contains at least one ("human", ...) turn.' + ) + if not system_str: + self.config.logger.debug( + "No system message in template for step '%s'; " + "passing empty string to adapter.", + step_name, + ) + response: LLMResponse = await provider.generate( + system=system_str, + human=human_str, + config=GenerateConfig( + temperature=prompt_settings.temperature, + model=prompt_settings.model, + ), + ) + except EvaluatorError: + raise + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: + raise wrap_provider_error( + e, + provider=prompt_settings.provider_type, + model=prompt_settings.model, + ) from e + if response.input_tokens is not None or response.output_tokens is not None: + token_usage = TokenUsage( + provider_type=prompt_settings.provider_type, + model=response.model, + input_tokens=response.input_tokens or 0, + output_tokens=response.output_tokens or 0, + ) + if parser_output_type is None: + return response.content + return _parse_json_output( + response.content, + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + response.model, + ) + + try: + return await self.execute_step( + step_name, + evaluation_metadata, + _run_via_provider, + extras={ + PROMPT_STEP_EXTRA_PROMPT_SETTINGS: prompt_settings_to_extras_value( + prompt_settings + ), + }, + ) + finally: + if token_usage is not None: + self.update_total_token_usage(token_usage, evaluation_metadata) + step = evaluation_metadata.step_details.get(step_name) + if step is not None: + step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE] = token_usage.model_dump( + mode="json" + ) + + # ── LangChain path (default, unchanged) ─────────────────────────── async def _run_chain() -> BaseModel | str: nonlocal token_usage try: @@ -342,29 +513,13 @@ async def _run_chain() -> BaseModel | str: from langchain_core.output_parsers.json import JsonOutputParser if json_dict_normalizer is not None: - loose = JsonOutputParser() - parsed_dict = await loose.ainvoke(ai_message) - if not isinstance(parsed_dict, dict): - # JSON parsed cleanly but the top-level value isn't an object - # (e.g. the LLM returned a JSON array or scalar). That's an - # output-shape failure, not a parse failure — surface it as - # OutputValidationError so callers can treat it consistently - # with schema-mismatch errors, and avoid the TypeError that - # ``dict(parsed_dict)`` would raise on a non-dict. - raise OutputValidationError( - "Model output is not a JSON object", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) - try: - normalized = json_dict_normalizer(parsed_dict) - except (TypeError, ValueError) as norm_err: - raise OutputValidationError( - "Model output could not be normalized before validation", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) from norm_err - return parser_output_type.model_validate(normalized) + return _parse_json_output( + str(ai_message.content), + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + prompt_settings.model, + ) parser = JsonOutputParser(pydantic_object=parser_output_type) raw = await parser.ainvoke(ai_message) diff --git a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py index 27caf004..bb41b809 100644 --- a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py @@ -33,6 +33,11 @@ InputSpec, TextInputSpec, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -81,5 +86,8 @@ "TextInputField", "TokenUsage", "InputValidationError", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "prompt_settings_to_extras_value", ] diff --git a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py new file mode 100644 index 00000000..0ccf489b --- /dev/null +++ b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py @@ -0,0 +1,133 @@ +"""LLM provider protocol and associated types for framework-agnostic model injection. + +These types define the interface that evaluation frameworks (Inspect AI, Braintrust, +Arize/Phoenix, Langfuse, etc.) implement to provide their own model execution to the SDK. + +``LLMGeneratorProtocol`` is a structural protocol (``typing.Protocol``) — integration +packages do not need to import or inherit from it. Any class with the correct ``generate`` +signature satisfies the protocol automatically for static type checkers. + +Response fields are aligned with OpenTelemetry GenAI semantic conventions: +https://opentelemetry.io/docs/specs/semconv/gen-ai/ +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import NamedTuple, Protocol + + +class LLMResponse(NamedTuple): + """Structured response from an LLM generation call. + + A ``NamedTuple`` — immutable, constructible by keyword or position, and + iterable for adapters that need to destructure the result. + + Fields are aligned with OpenTelemetry GenAI semantic conventions so that + observability adapters (Arize/Phoenix, Langfuse) can populate their spans + without additional parsing. + + Required fields (``content``, ``model``) must always be populated. + Optional fields should be populated whenever the underlying provider returns them. + """ + + content: str + """The model's text response.""" + + model: str + """The model that generated the response (``gen_ai.response.model``).""" + + input_tokens: int | None = None + """Number of input/prompt tokens consumed (``gen_ai.usage.input_tokens``).""" + + output_tokens: int | None = None + """Number of output/completion tokens generated (``gen_ai.usage.output_tokens``).""" + + +@dataclass +class GenerateConfig: + """Configuration for a single LLM generation call. + + All fields are optional. Adapters should apply whatever the underlying + provider supports and ignore the rest. + """ + + temperature: float | None = None + """Sampling temperature. 0.0 for deterministic output (recommended for evals).""" + + max_tokens: int | None = None + """Maximum number of tokens to generate.""" + + model: str | None = None + """Model identifier to request from the provider (e.g. ``"claude-opus-4-8"``). + + Adapters should use this when set and ignore it otherwise — the contract is + identical to all other ``GenerateConfig`` fields. When ``None`` (default), + the adapter uses whatever model it was constructed with. + + Populated from ``PromptSettings.model`` on the protocol path so that + adapter authors can inspect which model the evaluator expects without + reaching into ``prompt_settings`` directly. + """ + + +class LLMGeneratorProtocol(Protocol): + """Structural protocol for LLM generation adapters. + + Implement this protocol in an integration package to allow the SDK to call + your framework's model system. + + No import of this class is required in the implementing package. Structural + conformance (correct ``generate`` signature) is sufficient for static type + checkers. + + **Lifecycle**: if your adapter holds a connection pool or HTTP session, add + ``async def aclose(self) -> None`` and call it when done. The protocol does + not include ``aclose`` so that stateless adapters remain fully conformant + without boilerplate. Callers that want to support teardown should use + ``hasattr(adapter, "aclose")``. + + Example:: + + from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMResponse, + ) + + class MyFrameworkAdapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + response = await my_framework.call(system, human) + return LLMResponse( + content=response.text, + model=response.model_name, + input_tokens=response.usage.input, + output_tokens=response.usage.output, + ) + """ + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + """Generate a response from the LLM. + + Args: + system: The system prompt. + human: The human/user prompt. + config: Optional generation configuration. Adapters apply whatever + fields the underlying provider supports and ignore the rest. + + Returns: + ``LLMResponse`` with at minimum ``content`` and ``model`` populated. + Populate optional fields (token counts) whenever the provider returns them. + """ + ... diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index f595af9c..863d36e0 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -857,3 +857,295 @@ def passthrough(d: dict) -> dict: assert "JSON object" in str(exc_info.value) assert exc_info.value.provider is LLMProvider.GOOGLE assert exc_info.value.model == "gemini-2.0-flash" + + +# --------------------------------------------------------------------------- +# execute_prompt_chain_step — protocol path (llm_provider injected) +# --------------------------------------------------------------------------- + + +from learning_commons_evaluators.schemas.llm_provider import LLMResponse # noqa: E402 + + +def _make_adapter( + content: str, + model: str = "test-model", + input_tokens: int | None = 10, + output_tokens: int | None = 5, +) -> AsyncMock: + """Minimal mock that satisfies LLMGeneratorProtocol.generate().""" + adapter = AsyncMock() + adapter.generate = AsyncMock( + return_value=LLMResponse( + content=content, model=model, input_tokens=input_tokens, output_tokens=output_tokens + ) + ) + return adapter + + +_PROTO_SETTINGS = PromptSettings( + provider_type=LLMProvider.ANTHROPIC, + model="claude-opus-4-8", + temperature=0.0, +) + +_PROTO_TEMPLATE = ChatPromptTemplate.from_messages( + [("system", "You are a grader."), ("human", "{input}")] +) + + +class TestExecutePromptChainStepProtocolPath: + """Protocol path: llm_provider injected — LangChain provider is never called.""" + + def _ev(self, adapter: AsyncMock) -> _StubEvaluator: + return _StubEvaluator(create_config_no_telemetry(), llm_provider=adapter) + + async def test_returns_raw_string_when_parser_type_is_none(self, evaluation_metadata): + ev = self._ev(_make_adapter("plain prose")) + out = await ev.execute_prompt_chain_step( + step_name="raw", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=None, + ) + assert out == "plain prose" + + async def test_parses_clean_json(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(result, _ChainOutput) + assert result.label == "ok" + assert result.score == 7 + + async def test_strips_markdown_fences(self, evaluation_metadata): + fenced = f"```json\n{_CHAIN_JSON}\n```" + ev = self._ev(_make_adapter(fenced)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_trailing_prose(self, evaluation_metadata): + with_prose = f"{_CHAIN_JSON}\n\nHere is my reasoning for this score." + ev = self._ev(_make_adapter(with_prose)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_leading_prose(self, evaluation_metadata): + with_prefix = f"Here is the result:\n{_CHAIN_JSON}" + ev = self._ev(_make_adapter(with_prefix)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_json_dict_normalizer_path(self, evaluation_metadata): + class _Out(BaseModel): + n: int + doubled: int + + ev = self._ev(_make_adapter('{"n": 3}')) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: {**d, "doubled": d["n"] * 2}, + ) + assert result.n == 3 + assert result.doubled == 6 + + async def test_non_dict_json_in_normalizer_path_raises_output_validation_error( + self, evaluation_metadata + ): + class _Out(BaseModel): + n: int + + ev = self._ev(_make_adapter('["not", "an", "object"]')) + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: d, + ) + assert "JSON object" in str(exc_info.value) + + async def test_malformed_json_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter("not json at all")) + with pytest.raises(OutputValidationError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_schema_mismatch_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter('{"label": "only"}')) # missing required `score` + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, PydanticValidationError) + + async def test_token_usage_recorded_in_step_extras_and_total(self, evaluation_metadata): + ev = self._ev( + _make_adapter(_CHAIN_JSON, model="claude-opus-4-8", input_tokens=42, output_tokens=17) + ) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + step = evaluation_metadata.step_details["main"] + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["input_tokens"] == 42 + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["output_tokens"] == 17 + assert evaluation_metadata.total_token_usage[LLMProvider.ANTHROPIC].input_tokens == 42 + + async def test_token_usage_absent_when_llm_response_has_none_tokens(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON, input_tokens=None, output_tokens=None)) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert not evaluation_metadata.total_token_usage + + async def test_provider_error_wrapped_as_api_error(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=RuntimeError("network timeout")) + ev = self._ev(adapter) + with pytest.raises(APIError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, RuntimeError) + + async def test_evaluator_error_from_provider_reraises_unchanged(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=EvaluatorError("already wrapped")) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError, match="already wrapped"): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_keyboard_interrupt_from_provider_propagates(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=KeyboardInterrupt) + ev = self._ev(adapter) + with pytest.raises(KeyboardInterrupt): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_adapter_called_with_formatted_system_and_human(self, evaluation_metadata): + """Template formatting actually reaches the adapter with the correct strings.""" + from unittest.mock import ANY + + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with( + system="You are a grader.", + human="Hello", + config=ANY, + ) + + async def test_human_only_template_passes_empty_system(self, evaluation_metadata): + """Templates with no system turn pass empty string to the adapter without error.""" + from unittest.mock import ANY + + human_only = ChatPromptTemplate.from_messages([("human", "{input}")]) + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=human_only, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with(system="", human="Hello", config=ANY) + + async def test_template_with_missing_variable_raises_evaluator_error(self, evaluation_metadata): + """A missing template variable becomes an EvaluatorError, not a bare KeyError.""" + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={}, # missing required "input" variable + parser_output_type=_ChainOutput, + ) diff --git a/sdks/python/tests/schemas/test_llm_provider.py b/sdks/python/tests/schemas/test_llm_provider.py new file mode 100644 index 00000000..b0782097 --- /dev/null +++ b/sdks/python/tests/schemas/test_llm_provider.py @@ -0,0 +1,105 @@ +"""Tests for LLMGeneratorProtocol, LLMResponse, and GenerateConfig.""" + +from __future__ import annotations + +import learning_commons_evaluators +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class TestLLMResponse: + def test_required_fields(self): + r = LLMResponse(content="hello", model="anthropic/claude-opus-4-8") + assert r.content == "hello" + assert r.model == "anthropic/claude-opus-4-8" + assert r.input_tokens is None + assert r.output_tokens is None + + def test_optional_token_fields(self): + r = LLMResponse(content="text", model="test", input_tokens=100, output_tokens=50) + assert r.input_tokens == 100 + assert r.output_tokens == 50 + + +class TestGenerateConfig: + def test_defaults(self): + c = GenerateConfig() + assert c.temperature is None + assert c.max_tokens is None + + def test_with_values(self): + c = GenerateConfig(temperature=0.0, max_tokens=512) + assert c.temperature == 0.0 + assert c.max_tokens == 512 + + +class TestLLMGeneratorProtocol: + async def test_generate_returns_llm_response(self): + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + return LLMResponse(content="hi", model="m", input_tokens=5, output_tokens=2) + + result = await Adapter().generate(system="sys", human="hello") + assert result.content == "hi" + assert result.model == "m" + assert result.input_tokens == 5 + + async def test_generate_config_passed_through(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + cfg = GenerateConfig(temperature=0.0, max_tokens=256) + await Adapter().generate(system="sys", human="hello", config=cfg) + assert received[0] is cfg + assert received[0].temperature == 0.0 + + async def test_generate_config_none_by_default(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + await Adapter().generate(system="sys", human="hello") + assert received[0] is None + + def test_structural_conformance(self): + """Static type checkers validate this assignment — no subclassing required. + + LLMGeneratorProtocol is not @runtime_checkable; isinstance() is not available. + Conformance is enforced at type-check time (mypy/pyright) by the annotation below. + """ + + class Adapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + return LLMResponse(content="", model="test") + + # mypy/pyright validate this structurally + _adapter: LLMGeneratorProtocol = Adapter() + assert _adapter is not None # runtime no-op; static check is the value + + +def test_exported_from_package(): + assert "GenerateConfig" in learning_commons_evaluators.__all__ + assert "LLMGeneratorProtocol" in learning_commons_evaluators.__all__ + assert "LLMResponse" in learning_commons_evaluators.__all__