diff --git a/.gitignore b/.gitignore index ff15832..f790344 100644 --- a/.gitignore +++ b/.gitignore @@ -208,3 +208,7 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ +*.zip +/ECPBrain +/worktrees +/logs diff --git a/examples/customer_support_demo/manifest.yaml b/examples/customer_support_demo/manifest.yaml index 94ace1a..ed971eb 100644 --- a/examples/customer_support_demo/manifest.yaml +++ b/examples/customer_support_demo/manifest.yaml @@ -15,6 +15,9 @@ scenarios: field: evaluation_context condition: contains value: "refund_eligible=True" + - type: llm_judge + field: public_output + prompt: "Does the output clearly state if the order is eligible for a refund? Must confirm eligibility." - type: tool_usage tool_name: "lookup_order" arguments: diff --git a/runtime/python/README.md b/runtime/python/README.md index 4bec543..68339e6 100644 --- a/runtime/python/README.md +++ b/runtime/python/README.md @@ -7,7 +7,7 @@ ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and eval ## Install ```bash -pip install "ecp-runtime==0.3.1" +pip install "ecp-runtime==0.4.1" ``` ## Usage diff --git a/runtime/python/pyproject.toml b/runtime/python/pyproject.toml index b678524..d03364f 100644 --- a/runtime/python/pyproject.toml +++ b/runtime/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ecp-runtime" -version = "0.3.1" +version = "0.4.1" description = "Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context." authors = [ { name = "ECP Maintainers", email = "aniket.wattamwar17@gmail.com" }, @@ -29,4 +29,7 @@ Issues = "https://github.com/evaluation-context-protocol/ecp/issues" [project.scripts] ecp = "ecp_runtime.cli:app" +[project.entry-points.pytest11] +ecp_runtime = "ecp_runtime.pytest_plugin" + diff --git a/runtime/python/src/ecp_runtime/__init__.py b/runtime/python/src/ecp_runtime/__init__.py index 8278a8e..b2972b1 100644 --- a/runtime/python/src/ecp_runtime/__init__.py +++ b/runtime/python/src/ecp_runtime/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.3.1" +__version__ = "0.4.1" diff --git a/runtime/python/src/ecp_runtime/cli.py b/runtime/python/src/ecp_runtime/cli.py index e12ac23..109109f 100644 --- a/runtime/python/src/ecp_runtime/cli.py +++ b/runtime/python/src/ecp_runtime/cli.py @@ -1,4 +1,4 @@ -import glob +import glob import json import logging import os @@ -10,18 +10,11 @@ import typer from pydantic import ValidationError +# Import local modules (Using relative imports) from .reporter import HTMLReporter from .trend import RunTrendAnalyzer - -# Import local modules (Using relative imports) -try: - from .manifest import ECPManifest - from .runner import ECPRunner -except ImportError: - # Fallback for direct execution debugging - sys.path.append(os.path.dirname(__file__)) - from manifest import ECPManifest - from runner import ECPRunner +from .manifest import ECPManifest +from .runner import ECPRunner app = typer.Typer( name="ecp", @@ -81,6 +74,11 @@ def run( "--json", help="Print the JSON report to stdout", ), + export: Optional[str] = typer.Option( + None, + "--export", + help="Export evaluation results to an external platform (e.g., langsmith)", + ), fail_on_error: bool = typer.Option( True, "--fail-on-error/--no-fail-on-error", @@ -130,6 +128,30 @@ def run( if print_json: typer.echo(json.dumps(report_payload, indent=2)) + if export: + if export.lower() == "langsmith": + logger.info("Exporting results to LangSmith...") + try: + from langsmith import Client # type: ignore + client = Client() + project_name = config.name + for scenario in result_summary.get("scenarios", []): + for step in scenario.get("steps", []): + client.create_run( + name=scenario.get("name"), + run_type="chain", + inputs={"input": step.get("input")}, + outputs={"output": step.get("output")}, + project_name=project_name + ) + logger.info("Successfully exported to LangSmith.") + except ImportError: + logger.warning("langsmith package not installed. Skipping export.") + except Exception as e: + logger.error("Failed to export to LangSmith: %s", e) + else: + logger.warning("Unsupported export target: %s", export) + if fail_on_error and failed > 0: raise typer.Exit(code=2) diff --git a/runtime/python/src/ecp_runtime/graders.py b/runtime/python/src/ecp_runtime/graders.py index 1c5de40..1474d91 100644 --- a/runtime/python/src/ecp_runtime/graders.py +++ b/runtime/python/src/ecp_runtime/graders.py @@ -2,13 +2,18 @@ import re from typing import Any, Dict, List, Tuple -from .manifest import GraderConfig, StepConfig +try: + from .manifest import GraderConfig, StepConfig +except ImportError: + import sys, os + sys.path.append(os.path.dirname(__file__)) + from manifest import GraderConfig, StepConfig # type: ignore # Try importing OpenAI, but don't crash if it's missing (unless used) try: - from openai import OpenAI + from openai import OpenAI # type: ignore except ImportError: - OpenAI = None + OpenAI = None # type: ignore def _llm_judge_model() -> str: diff --git a/runtime/python/src/ecp_runtime/manifest.py b/runtime/python/src/ecp_runtime/manifest.py index 8d6faa0..946787c 100644 --- a/runtime/python/src/ecp_runtime/manifest.py +++ b/runtime/python/src/ecp_runtime/manifest.py @@ -1,3 +1,6 @@ +import os +import csv +import json from typing import Any, Dict, List, Literal, Optional import yaml @@ -46,9 +49,22 @@ class StepConfig(BaseModel): constraints: Dict[str, Any] = Field(default_factory=dict) graders: List[GraderConfig] = Field(default_factory=list) +class DatasetConfig(BaseModel): + type: Literal["csv", "jsonl"] + source: str + input_column: str = "input" + output_column: str = "output" + class ScenarioConfig(BaseModel): name: str - steps: List[StepConfig] + steps: List[StepConfig] = Field(default_factory=list) + dataset: Optional[DatasetConfig] = None + + @model_validator(mode="after") + def _validate_steps_or_dataset(self) -> "ScenarioConfig": + if not self.steps and not self.dataset: + raise ValueError("Scenario requires either 'steps' or 'dataset'") + return self # --- The Root Manifest Schema --- class ECPManifest(BaseModel): @@ -63,4 +79,45 @@ def from_yaml(cls, path: str) -> "ECPManifest": data = yaml.safe_load(f) if not isinstance(data, dict): raise ValueError("Manifest YAML root must be an object") - return cls(**data) + manifest = cls(**data) + base_dir = os.path.dirname(os.path.abspath(path)) + manifest._resolve_datasets(base_dir) + return manifest + + def _resolve_datasets(self, base_dir: str): + for scenario in self.scenarios: + if scenario.dataset and not scenario.steps: + source_path = os.path.join(base_dir, scenario.dataset.source) + if not os.path.exists(source_path): + raise ValueError(f"Dataset file not found: {source_path}") + + if scenario.dataset.type == "csv": + with open(source_path, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + input_text = row.get(scenario.dataset.input_column, "") + output_text = row.get(scenario.dataset.output_column, "") + scenario.steps.append(StepConfig( + input=input_text, + graders=[GraderConfig( + type="text_match", + condition="equals", + value=output_text + )] if output_text else [] + )) + elif scenario.dataset.type == "jsonl": + with open(source_path, "r", encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + row = json.loads(line) + input_text = row.get(scenario.dataset.input_column, "") + output_text = row.get(scenario.dataset.output_column, "") + scenario.steps.append(StepConfig( + input=input_text, + graders=[GraderConfig( + type="text_match", + condition="equals", + value=output_text + )] if output_text else [] + )) diff --git a/runtime/python/src/ecp_runtime/pytest_plugin.py b/runtime/python/src/ecp_runtime/pytest_plugin.py new file mode 100644 index 0000000..483149b --- /dev/null +++ b/runtime/python/src/ecp_runtime/pytest_plugin.py @@ -0,0 +1,74 @@ +import os +import pytest +from typing import Any, Dict + +from .runner import _create_agent + +def pytest_addoption(parser): + parser.addoption( + "--ecp-target", + action="store", + default=None, + help="The target command or HTTP URL for the ECP agent" + ) + +class ECPAgentFixture: + def __init__(self, target: str, rpc_timeout: float = 30.0): + self.target = target + self.rpc_timeout = rpc_timeout + self._agent = None + + def start(self): + self._agent = _create_agent(self.target, self.rpc_timeout) + self._agent.start() + resp = self._agent.send_rpc("agent/initialize", {"config": {}}) + self._ensure_rpc_success(resp, "agent/initialize") + return self + + def stop(self): + if self._agent: + self._agent.stop() + self._agent = None + + def step(self, input_text: str) -> Dict[str, Any]: + resp = self._agent.send_rpc("agent/step", {"input": input_text}) + self._ensure_rpc_success(resp, "agent/step") + return resp.get("result", {}) + + def reset(self) -> Dict[str, Any]: + resp = self._agent.send_rpc("agent/reset", {}) + self._ensure_rpc_success(resp, "agent/reset") + return resp.get("result", {}) + + def _ensure_rpc_success(self, rpc_resp: Dict[str, Any], method: str) -> None: + if "error" not in rpc_resp: + return + error = rpc_resp.get("error") or {} + code = error.get("code") + message = error.get("message", "Unknown JSON-RPC error") + raise RuntimeError( + f"RPC call failed ({method}): code={code}, message={message}" + ) + +@pytest.fixture +def ecp_agent(request): + target = request.config.getoption("--ecp-target") + marker = request.node.get_closest_marker("ecp") + if marker and "target" in marker.kwargs: + target = marker.kwargs["target"] + + if not target: + pytest.skip("No ECP target specified. Use --ecp-target or @pytest.mark.ecp(target=...)") + + rpc_timeout = float(os.environ.get("ECP_RPC_TIMEOUT", "30")) + agent = ECPAgentFixture(target, rpc_timeout=rpc_timeout) + agent.start() + + yield agent + + agent.stop() + +def pytest_configure(config): + config.addinivalue_line( + "markers", "ecp: marks tests that require an ECP agent" + ) diff --git a/runtime/python/src/ecp_runtime/runner.py b/runtime/python/src/ecp_runtime/runner.py index 97be2be..8168091 100644 --- a/runtime/python/src/ecp_runtime/runner.py +++ b/runtime/python/src/ecp_runtime/runner.py @@ -1,4 +1,4 @@ -""" +""" Docstring for runtime.python.src.ecp_runtime.runner Simplified Version. V0.1 @@ -17,7 +17,12 @@ from urllib import error, request from urllib.parse import urlparse -from .graders import evaluate_step +try: + from .graders import evaluate_step +except ImportError: + import sys, os + sys.path.append(os.path.dirname(__file__)) + from graders import evaluate_step # type: ignore logger = logging.getLogger(__name__) diff --git a/schema/manifest.schema.json b/schema/manifest.schema.json index 2d39c71..2241a43 100644 --- a/schema/manifest.schema.json +++ b/schema/manifest.schema.json @@ -61,9 +61,35 @@ }, "required": ["input"] } + }, + "dataset": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": ["csv", "jsonl"] + }, + "source": { + "type": "string" + }, + "input_column": { + "type": "string", + "default": "input" + }, + "output_column": { + "type": "string", + "default": "output" + } + }, + "required": ["type", "source"] } }, - "required": ["name", "steps"] + "required": ["name"], + "anyOf": [ + {"required": ["steps"]}, + {"required": ["dataset"]} + ] } } },