feat: pytest, dataset_loader, --export to langsmith

.gitignore

@@ -208,3 +208,7 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+*.zip
+/ECPBrain
+/worktrees
+/logs

examples/customer_support_demo/manifest.yaml

@@ -15,6 +15,9 @@ scenarios:
             field: evaluation_context
             condition: contains
             value: "refund_eligible=True"
+          - type: llm_judge
+            field: public_output
+            prompt: "Does the output clearly state if the order is eligible for a refund? Must confirm eligibility."
           - type: tool_usage
             tool_name: "lookup_order"
             arguments:

runtime/python/README.md

@@ -7,7 +7,7 @@ ECP is a vendor-neutral protocol for testing agent outputs, tool calls, and eval
 ## Install
 
 ```bash
-pip install "ecp-runtime==0.3.1"
+pip install "ecp-runtime==0.4.1"
 ```
 
 ## Usage

runtime/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ecp-runtime"
-version = "0.3.1"
+version = "0.4.1"
 description = "Vendor-neutral runtime for portable AI agent evaluations with outputs, tool calls, and audit context."
 authors = [
   { name = "ECP Maintainers", email = "aniket.wattamwar17@gmail.com" },
@@ -29,4 +29,7 @@ Issues = "https://github.com/evaluation-context-protocol/ecp/issues"
 [project.scripts]
 ecp = "ecp_runtime.cli:app" 
 
+[project.entry-points.pytest11]
+ecp_runtime = "ecp_runtime.pytest_plugin"
+
 

runtime/python/src/ecp_runtime/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.3.1"
+__version__ = "0.4.1"
 

runtime/python/src/ecp_runtime/cli.py

@@ -1,4 +1,4 @@
-import glob
+import glob
 import json
 import logging
 import os
@@ -10,18 +10,11 @@
 import typer
 from pydantic import ValidationError
 
+# Import local modules (Using relative imports)
 from .reporter import HTMLReporter
 from .trend import RunTrendAnalyzer
-
-# Import local modules (Using relative imports)
-try:
-    from .manifest import ECPManifest
-    from .runner import ECPRunner
-except ImportError:
-    # Fallback for direct execution debugging
-    sys.path.append(os.path.dirname(__file__))
-    from manifest import ECPManifest
-    from runner import ECPRunner
+from .manifest import ECPManifest
+from .runner import ECPRunner 
 
 app = typer.Typer(
     name="ecp",
@@ -81,6 +74,11 @@ def run(
         "--json",
         help="Print the JSON report to stdout",
     ),
+    export: Optional[str] = typer.Option(
+        None,
+        "--export",
+        help="Export evaluation results to an external platform (e.g., langsmith)",
+    ),
     fail_on_error: bool = typer.Option(
         True,
         "--fail-on-error/--no-fail-on-error",
@@ -130,6 +128,30 @@ def run(
         if print_json:
             typer.echo(json.dumps(report_payload, indent=2))
 
+        if export:
+            if export.lower() == "langsmith":
+                logger.info("Exporting results to LangSmith...")
+                try:
+                    from langsmith import Client  # type: ignore
+                    client = Client()
+                    project_name = config.name
+                    for scenario in result_summary.get("scenarios", []):
+                        for step in scenario.get("steps", []):
+                            client.create_run(
+                                name=scenario.get("name"),
+                                run_type="chain",
+                                inputs={"input": step.get("input")},
+                                outputs={"output": step.get("output")},
+                                project_name=project_name
+                            )
+                    logger.info("Successfully exported to LangSmith.")
+                except ImportError:
+                    logger.warning("langsmith package not installed. Skipping export.")
+                except Exception as e:
+                    logger.error("Failed to export to LangSmith: %s", e)
+            else:
+                logger.warning("Unsupported export target: %s", export)
+
         if fail_on_error and failed > 0:
             raise typer.Exit(code=2)
 

runtime/python/src/ecp_runtime/graders.py

@@ -2,13 +2,18 @@
 import re
 from typing import Any, Dict, List, Tuple
 
-from .manifest import GraderConfig, StepConfig
+try:
+    from .manifest import GraderConfig, StepConfig
+except ImportError:
+    import sys, os
+    sys.path.append(os.path.dirname(__file__))
+    from manifest import GraderConfig, StepConfig  # type: ignore
 
 # Try importing OpenAI, but don't crash if it's missing (unless used)
 try:
-    from openai import OpenAI
+    from openai import OpenAI  # type: ignore
 except ImportError:
-    OpenAI = None
+    OpenAI = None  # type: ignore
 
 
 def _llm_judge_model() -> str:

runtime/python/src/ecp_runtime/manifest.py

@@ -1,3 +1,6 @@
+import os
+import csv
+import json
 from typing import Any, Dict, List, Literal, Optional
 
 import yaml
@@ -46,9 +49,22 @@ class StepConfig(BaseModel):
     constraints: Dict[str, Any] = Field(default_factory=dict)
     graders: List[GraderConfig] = Field(default_factory=list)
 
+class DatasetConfig(BaseModel):
+    type: Literal["csv", "jsonl"]
+    source: str
+    input_column: str = "input"
+    output_column: str = "output"
+
 class ScenarioConfig(BaseModel):
     name: str
-    steps: List[StepConfig]
+    steps: List[StepConfig] = Field(default_factory=list)
+    dataset: Optional[DatasetConfig] = None
+
+    @model_validator(mode="after")
+    def _validate_steps_or_dataset(self) -> "ScenarioConfig":
+        if not self.steps and not self.dataset:
+            raise ValueError("Scenario requires either 'steps' or 'dataset'")
+        return self
 
 # --- The Root Manifest Schema ---
 class ECPManifest(BaseModel):
@@ -63,4 +79,45 @@ def from_yaml(cls, path: str) -> "ECPManifest":
             data = yaml.safe_load(f)
         if not isinstance(data, dict):
             raise ValueError("Manifest YAML root must be an object")
-        return cls(**data)
+        manifest = cls(**data)
+        base_dir = os.path.dirname(os.path.abspath(path))
+        manifest._resolve_datasets(base_dir)
+        return manifest
+
+    def _resolve_datasets(self, base_dir: str):
+        for scenario in self.scenarios:
+            if scenario.dataset and not scenario.steps:
+                source_path = os.path.join(base_dir, scenario.dataset.source)
+                if not os.path.exists(source_path):
+                    raise ValueError(f"Dataset file not found: {source_path}")
+
+                if scenario.dataset.type == "csv":
+                    with open(source_path, "r", encoding="utf-8") as f:
+                        reader = csv.DictReader(f)
+                        for row in reader:
+                            input_text = row.get(scenario.dataset.input_column, "")
+                            output_text = row.get(scenario.dataset.output_column, "")
+                            scenario.steps.append(StepConfig(
+                                input=input_text,
+                                graders=[GraderConfig(
+                                    type="text_match",
+                                    condition="equals",
+                                    value=output_text
+                                )] if output_text else []
+                            ))
+                elif scenario.dataset.type == "jsonl":
+                    with open(source_path, "r", encoding="utf-8") as f:
+                        for line in f:
+                            if not line.strip():
+                                continue
+                            row = json.loads(line)
+                            input_text = row.get(scenario.dataset.input_column, "")
+                            output_text = row.get(scenario.dataset.output_column, "")
+                            scenario.steps.append(StepConfig(
+                                input=input_text,
+                                graders=[GraderConfig(
+                                    type="text_match",
+                                    condition="equals",
+                                    value=output_text
+                                )] if output_text else []
+                            ))

runtime/python/src/ecp_runtime/pytest_plugin.py

@@ -0,0 +1,74 @@
+import os
+import pytest
+from typing import Any, Dict
+
+from .runner import _create_agent
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--ecp-target",
+        action="store",
+        default=None,
+        help="The target command or HTTP URL for the ECP agent"
+    )
+
+class ECPAgentFixture:
+    def __init__(self, target: str, rpc_timeout: float = 30.0):
+        self.target = target
+        self.rpc_timeout = rpc_timeout
+        self._agent = None
+
+    def start(self):
+        self._agent = _create_agent(self.target, self.rpc_timeout)
+        self._agent.start()
+        resp = self._agent.send_rpc("agent/initialize", {"config": {}})
+        self._ensure_rpc_success(resp, "agent/initialize")
+        return self
+
+    def stop(self):
+        if self._agent:
+            self._agent.stop()
+            self._agent = None
+
+    def step(self, input_text: str) -> Dict[str, Any]:
+        resp = self._agent.send_rpc("agent/step", {"input": input_text})
+        self._ensure_rpc_success(resp, "agent/step")
+        return resp.get("result", {})
+
+    def reset(self) -> Dict[str, Any]:
+        resp = self._agent.send_rpc("agent/reset", {})
+        self._ensure_rpc_success(resp, "agent/reset")
+        return resp.get("result", {})
+
+    def _ensure_rpc_success(self, rpc_resp: Dict[str, Any], method: str) -> None:
+        if "error" not in rpc_resp:
+            return
+        error = rpc_resp.get("error") or {}
+        code = error.get("code")
+        message = error.get("message", "Unknown JSON-RPC error")
+        raise RuntimeError(
+            f"RPC call failed ({method}): code={code}, message={message}"
+        )
+
+@pytest.fixture
+def ecp_agent(request):
+    target = request.config.getoption("--ecp-target")
+    marker = request.node.get_closest_marker("ecp")
+    if marker and "target" in marker.kwargs:
+        target = marker.kwargs["target"]
+
+    if not target:
+        pytest.skip("No ECP target specified. Use --ecp-target or @pytest.mark.ecp(target=...)")
+
+    rpc_timeout = float(os.environ.get("ECP_RPC_TIMEOUT", "30"))
+    agent = ECPAgentFixture(target, rpc_timeout=rpc_timeout)
+    agent.start()
+
+    yield agent
+
+    agent.stop()
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ecp: marks tests that require an ECP agent"
+    )

runtime/python/src/ecp_runtime/runner.py

@@ -1,4 +1,4 @@
-"""
+"""
 Docstring for runtime.python.src.ecp_runtime.runner
 
 Simplified Version. V0.1
@@ -17,7 +17,12 @@
 from urllib import error, request
 from urllib.parse import urlparse
 
-from .graders import evaluate_step
+try:
+    from .graders import evaluate_step
+except ImportError:
+    import sys, os
+    sys.path.append(os.path.dirname(__file__))
+    from graders import evaluate_step  # type: ignore
 
 logger = logging.getLogger(__name__)
 

schema/manifest.schema.json

@@ -61,9 +61,35 @@
               },
               "required": ["input"]
             }
+          },
+          "dataset": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": ["csv", "jsonl"]
+              },
+              "source": {
+                "type": "string"
+              },
+              "input_column": {
+                "type": "string",
+                "default": "input"
+              },
+              "output_column": {
+                "type": "string",
+                "default": "output"
+              }
+            },
+            "required": ["type", "source"]
           }
         },
-        "required": ["name", "steps"]
+        "required": ["name"],
+        "anyOf": [
+          {"required": ["steps"]},
+          {"required": ["dataset"]}
+        ]
       }
     }
   },