Preparing to publish Python wheel, and moving python examples to python-examples/ top-level dir

mise.toml

@@ -1,7 +1,7 @@
 monorepo_root = true
 
 [monorepo]
-config_roots = [".", "cli", "typescript", "python"]
+config_roots = [".", "cli", "typescript", "python", "python-examples"]
 
 [tools]
 # keep this version inline with the version declared in .github/workflows/typescript

python-examples/.python-version

@@ -0,0 +1 @@
+3.12

python-examples/README.md

@@ -0,0 +1,10 @@
+# Quantiles Python SDK Examples
+
+This directory in the Quantiles monorepo contains examples to illustrate how to build [`custom_code` evals](https://quantiles.io/documentation/custom-evaluations) on the Quantiles platform using the [Python SDK](https://quantiles.io/documentation/reference/python-sdk).
+
+The following `custom_code` evals are available along with a [`quantiles.toml` configuration file](https://quantiles.io/documentation/configuration) to make them easily runnable. They are detailed below:
+
+| Eval | `qt run` command | Source file | Notes |
+| --- | --- | --- | --- |
+| PubMedQA | `qt run custom_pubmedqa` | [`src/pubmedqa.py`](./src/pubmedqa.py) | [PubMedQA](https://pubmedqa.github.io/) is a biomedical question-answering benchmark. This file shows how a custom evaluation can be implemented, but PubMedQA is also available as a built-in Quantiles benchmark: `qt run pubmedqa`. |
+| Example prompt eval | `qt run custom_prompt_eval` | [`src/prompt_eval.py`](./src/prompt_eval.py) | This is a [`custom_code`](https://quantiles.io/documentation/custom-evaluations) eval to illustrate how you might use Quantiles to evaluate the performance of a customer support chatbot with various system prompts |

python-examples/mise.toml

@@ -0,0 +1,5 @@
+[tasks.run-pubmedqa]
+run = "qt run custom_pubmedqa"
+
+[tasks.run_prompt_eval]
+run = "qt run custom_prompt_eval"

python-examples/pyproject.toml

@@ -0,0 +1,10 @@
+[project]
+name = "python-examples"
+version = "0.1.0"
+description = "Example custom evals using the Quantiles Python SDK"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "pydantic>=2.13.4",
+    "quantiles>=0.1.0,<1.0.0",
+]

python-examples/quantiles.toml

@@ -0,0 +1,10 @@
+# PubMedQA built with the Python SDK (not using the built-in benchmark)
+[benchmarks.custom_pubmedqa]
+type = "custom_code"
+command = ["uv", "run", "src/pubmedqa.py"]
+input = { model_name = "openai:gpt_5_nano", num_examples = 500 }
+
+# PubMedQA built with the Python SDK (not using the built-in benchmark)
+[benchmarks.custom_prompt_eval]
+type = "custom_code"
+command = ["uv", "run", "src/prompt_eval.py"]

python-examples/src/prompt_eval.py

@@ -0,0 +1,120 @@
+from typing import Literal, TypedDict, cast
+
+from quantiles import (
+    JsonValue,
+    Statistics,
+    emit,
+    entrypoint,
+    step,
+    workflow,
+)
+from quantiles.workflow_context import WorkflowContext
+
+Label = Literal["billing", "bug", "how_to"]
+
+
+class EvalInput(TypedDict):
+    prompt_version: str
+
+
+class EvalCase(TypedDict):
+    id: str
+    ticket: str
+    expected: Label
+
+
+class CaseResult(TypedDict):
+    id: str
+    expected: Label
+    prediction: Label
+    correct: bool
+    tokens_used: int
+
+
+CASES: list[EvalCase] = [
+    {
+        "id": "double-charge",
+        "ticket": "I was charged twice for my subscription renewal.",
+        "expected": "billing",
+    },
+    {
+        "id": "csv-crash",
+        "ticket": "The app crashes every time I upload a CSV file.",
+        "expected": "bug",
+    },
+    {
+        "id": "invite-teammate",
+        "ticket": "Can you show me how to invite another teammate?",
+        "expected": "how_to",
+    },
+]
+
+
+async def evaluate_case(
+    ctx: WorkflowContext,
+    item: EvalCase,
+    prompt_version: str,
+) -> CaseResult:
+    prediction = cast(
+        Label,
+        await step(
+            ctx,
+            step_key=f"case:{item['id']}",
+            input_value={
+                "sample_id": item["id"],
+                "ticket": item["ticket"],
+                "expected": item["expected"],
+                "prompt_version": prompt_version,
+            },
+            execute=lambda: call_model(item["ticket"]),
+        ),
+    )
+
+    return {
+        "id": item["id"],
+        "expected": item["expected"],
+        "prediction": prediction,
+        "correct": prediction == item["expected"],
+        "tokens_used": len(item["ticket"].split()),
+    }
+
+
+async def handler(input_value: EvalInput | None, ctx: WorkflowContext) -> JsonValue:
+    prompt_version = (input_value or {}).get("prompt_version", "A")
+    results: list[CaseResult] = []
+    for item in CASES:
+        results.append(await evaluate_case(ctx, item, prompt_version))
+
+    correct = sum(1 for result in results if result["correct"])
+    accuracy = Statistics.accuracy(correct, len(results))
+    tokens_used = sum(result["tokens_used"] for result in results)
+
+    await emit(ctx, "accuracy", accuracy)
+    await emit(ctx, "correct_count", float(correct))
+    await emit(ctx, "total_count", float(len(results)))
+    await emit(ctx, "tokens_used", float(tokens_used), "tokens")
+
+    return cast(
+        JsonValue,
+        # the below value is a valid JsonValue
+        {
+            "accuracy": accuracy,
+            "correct_count": correct,
+            "total_count": len(results),
+            "results": results,
+        },
+    )
+
+
+async def call_model(ticket: str) -> Label:
+    text = ticket.lower()
+    if "crash" in text:
+        return "bug"
+    if "invite" in text or "show me" in text:
+        return "how_to"
+    return "billing"
+
+
+if __name__ == "__main__":
+    support_triage = workflow("custom_prompt_eval", handler)
+    entrypoint(support_triage)