From ca99e9c0b41092292b732ab1417d26fb79be6500 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Thu, 25 Jun 2026 14:49:19 -0400 Subject: [PATCH 1/8] docs: reconcile demo evidence docs --- README.md | 18 +++++++++++------- demos/README.md | 8 +++++++- docs/demos-results.md | 35 ++++++++++++++++++++++------------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4c6e5d8..7c28ce0 100644 --- a/README.md +++ b/README.md @@ -134,12 +134,16 @@ compiler owns deterministic state transitions. ## Does it Work? -Yes, on the current scored demo set, compiler-mediated paths pass every scored -scenario. +Yes. The current demo suite in this repository contains 8 scored demos +(`01`-`05`, `07`, `08`, `09`) plus 1 informational demo (`06`). -- Scope: evaluated across **7 models** and **3 provider paths** (`ollama`, `openai`, `openai_compatible`). -- Scored checks (**6 demos per model**; Demo 6 excluded): baseline **26 / 42**, compiler **42 / 42**, compiler+compact **42 / 42**. -- Baseline behavior remains model-dependent. +A published historical cross-model matrix is available for an earlier scored +demo set. + +- Historical matrix scope: **7 models** and **3 provider paths** (`ollama`, `openai`, `openai_compatible`). +- Historical scored checks (**6 demos per model**; Demo 6 excluded): baseline **26 / 42**, compiler **42 / 42**, compiler+compact **42 / 42**. +- That published matrix was recorded against Context Compiler **0.6.15** and predates the later scored additions (`08` and `09`) plus the published `reinjected-state` comparison. +- Baseline behavior in those historical runs remained model-dependent. Interpretation guide: - Demos `01`-`05` and `07` focus on persistence and policy-following behavior. @@ -147,8 +151,8 @@ Interpretation guide: - Demos `08`/`09` show what prompt text does not implement by itself. - Plain reinjection can produce plausible responses, but it does not check whether replacement is allowed or wait for confirmation before saving changes. -→ [Full results and demo output](demos/README.md) -Canonical matrix: [docs/demos-results.md](docs/demos-results.md) +→ [Current demo set and output modes](demos/README.md) +Historical published matrix: [docs/demos-results.md](docs/demos-results.md) ## Interactive Playground diff --git a/demos/README.md b/demos/README.md index 5cffcc2..9307be1 100644 --- a/demos/README.md +++ b/demos/README.md @@ -142,7 +142,12 @@ is unavailable. ## Results -The canonical cross-model results matrix is maintained in [docs/demos-results.md](../docs/demos-results.md). +This README describes the current demo suite in this repository. + +The published cross-model matrix in [docs/demos-results.md](../docs/demos-results.md) +is historical evidence, not current release evidence. That matrix was recorded +against Context Compiler `0.6.15` and predates the later scored additions +`08` and `09`. Notes: - There are **8 scored demos** (`01`–`05`, `07`, `08`, `09`). `06_context_compaction` is informational and excluded from PASS/FAIL totals. @@ -156,6 +161,7 @@ Notes: - Demos `08` and `09` cover cases prompt text does not implement by itself, such as checking whether replacement is allowed and waiting for confirmation before saving changes. - Plain prompt reinjection can produce reasonable answers, but it does not run these checks by itself. - Similar outcomes across models in `08`/`09` reflect app behavior limits, not model leaderboard ranking. +- No current cross-model rerun is published here for the full 8-demo scored set. ### Demo 05 example (prompt drift under longer context) diff --git a/docs/demos-results.md b/docs/demos-results.md index c2cdfc4..3abc1be 100644 --- a/docs/demos-results.md +++ b/docs/demos-results.md @@ -1,19 +1,28 @@ -# Demo Results +# Historical Demo Results -Canonical reference for the current LLM demo matrix and methodology. +Historical LLM demo evidence for an earlier published matrix. -Note: this published matrix predates the `reinjected-state` path added in the 0.7.1 demo/evaluation experiment. It currently reports baseline/compiler/compiler+compact only. +This document is not the current release-evidence summary for the repository's +present demo suite. It preserves earlier published results so readers can see +what was measured at the time. -## Scope +The matrix below: + +- was recorded against Context Compiler `0.6.15` +- covers the earlier scored set `01`-`05`, `07` only +- predates the later scored additions `08` and `09` +- predates the published `reinjected-state` comparison in the cross-model matrix + +## Historical Scope - Scored demos: `01`, `02`, `03`, `04`, `05`, `07` (6 total) - Informational demo: `06_context_compaction` (excluded from PASS/FAIL totals) -Methodology note (2026-05): the demo suite now also includes scored demos `08` -and `09` for rules about when state is allowed to change. The published matrix below predates those additions and -has not yet been fully rerun with the expanded scored set. +Current repository note (2026-06): the demo suite now also includes scored demos +`08` and `09` for rules about when state is allowed to change. No published +cross-model rerun is included here for that expanded scored set. -## Results Matrix +## Historical Results Matrix | Provider Path | Model | Baseline (P/F) | Compiler (P/F) | Compiler+Compact (P/F) | | :-- | :-- | :--: | :--: | :--: | @@ -25,7 +34,7 @@ has not yet been fully rerun with the expanded scored set. | `openai_compatible` | `anthropic/claude-sonnet-4-5-20250929` | 4 / 2 | 6 / 0 | 6 / 0 | | `openai_compatible` | `anthropic/claude-opus-4-1-20250805` | 4 / 2 | 6 / 0 | 6 / 0 | -## Totals (Derived from Matrix) +## Historical Totals (Derived from Matrix) - Model runs: `7` - Scored demos per run: `6` @@ -37,7 +46,7 @@ Aggregate pass totals: - Compiler: `42 / 42` - Compiler+compact: `42 / 42` -## Methodology +## Historical Methodology Primary command: @@ -72,7 +81,7 @@ model outputs. The `reinjected-state` path is plain state text injection and does not include compiler checks like replacement preconditions or pending confirmation handling. -### Run metadata +### Historical Run Metadata - Date: 2026-05-06 - Context Compiler: 0.6.15 @@ -90,7 +99,7 @@ confirmation handling. - Plain prompt reinjection can produce reasonable answers, but it does not run these checks by itself. - Demos `08`/`09` are not general LLM quality benchmarks. Baseline and reinjected-state can produce plausible text and still `FAIL` when those app-side checks are missing. -## Demo 05 Long-Transcript Stress (Exploratory Frontier Runs) +## Historical Demo 05 Long-Transcript Stress (Exploratory Frontier Runs) Additional exploratory runs extended Demo 05 to higher transcript lengths for selected frontier models. These runs are separate from the full cross-model matrix above, which @@ -105,7 +114,7 @@ This is exploratory evidence, not benchmark authority. Reinjection can be enough in some persistence scenarios, while compiler-mediated paths still provide explicit state-change rules. -## Local Ollama Context-Size Sweep (0.7.1 Experiment) +## Historical Local Ollama Context-Size Sweep (0.7.1 Experiment) This section reports the refreshed local-only matrix with the `reinjected-state` path and explicit context-size ladder runs. Historical hosted-provider matrix From eaa9835ff415a4fd6d94940d7444c0d21fdbafdd Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Fri, 26 Jun 2026 01:25:36 -0400 Subject: [PATCH 2/8] docs: reconcile current and historical demo evidence --- demos/README.md | 10 +++---- docs/demos-results.md | 61 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/demos/README.md b/demos/README.md index 9307be1..e59709b 100644 --- a/demos/README.md +++ b/demos/README.md @@ -144,10 +144,11 @@ is unavailable. This README describes the current demo suite in this repository. -The published cross-model matrix in [docs/demos-results.md](../docs/demos-results.md) -is historical evidence, not current release evidence. That matrix was recorded -against Context Compiler `0.6.15` and predates the later scored additions -`08` and `09`. +The published results page in [docs/demos-results.md](../docs/demos-results.md) +includes: + +- a current 2026-06 frontier rerun for the full 8-demo scored set +- the older historical 0.6.15 matrix for the earlier 6-demo scored set Notes: - There are **8 scored demos** (`01`–`05`, `07`, `08`, `09`). `06_context_compaction` is informational and excluded from PASS/FAIL totals. @@ -161,7 +162,6 @@ Notes: - Demos `08` and `09` cover cases prompt text does not implement by itself, such as checking whether replacement is allowed and waiting for confirmation before saving changes. - Plain prompt reinjection can produce reasonable answers, but it does not run these checks by itself. - Similar outcomes across models in `08`/`09` reflect app behavior limits, not model leaderboard ranking. -- No current cross-model rerun is published here for the full 8-demo scored set. ### Demo 05 example (prompt drift under longer context) diff --git a/docs/demos-results.md b/docs/demos-results.md index 3abc1be..6c56547 100644 --- a/docs/demos-results.md +++ b/docs/demos-results.md @@ -1,8 +1,65 @@ -# Historical Demo Results +# Demo Results + +Published LLM demo evidence for this repository. + +This document preserves earlier published results and also records newer reruns +where available. + +## Current Frontier Rerun (2026-06) + +This rerun covers the current 8-demo scored set: + +- scored demos: `01`-`05`, `07`, `08`, `09` +- informational demo: `06_context_compaction` + +### Frontier Results Matrix + +| Provider Path | Model | Baseline (P/F) | Reinjected-state (P/F) | Compiler (P/F) | Compiler+Compact (P/F) | +| :-- | :-- | :--: | :--: | :--: | :--: | +| `openai` | `gpt-4.1` | 4 / 4 | 6 / 2 | 8 / 0 | 8 / 0 | +| `openai` | `gpt-5` | 3 / 5 | 6 / 2 | 8 / 0 | 8 / 0 | +| `openai_compatible` | `anthropic/claude-sonnet-4-6` | 3 / 5 | 5 / 3 | 8 / 0 | 8 / 0 | +| `openai_compatible` | `anthropic/claude-opus-4-1` | 4 / 4 | 6 / 2 | 8 / 0 | 8 / 0 | + +### Frontier Totals (Derived from Matrix) + +- Model runs: `4` +- Scored demos per run: `8` +- Aggregate scored checks per path: `32` + +Aggregate pass totals: + +- Baseline: `14 / 32` +- Reinjected-state: `23 / 32` +- Compiler: `32 / 32` +- Compiler+compact: `32 / 32` + +### Frontier Run Metadata + +- Date: 2026-06-26 +- Context Compiler: working tree before Python `0.8.0` release +- Command family: `uv run python -m demos.run_demo all` +- Artifact source: `/tmp/context-compiler-demo-results` + +### Frontier Notes + +- The current scored set adds `08` and `09`, which test app-side state-transition + rules rather than general response quality. +- A third-pass full rerun corrected the earlier `gpt-4.1` Demo `02` compiler + failure; the current published `gpt-4.1` row is now `8 / 0` on both compiler + paths. +- A fresh full rerun corrected an earlier stale `gpt-5` Demo `02` summary; the + current published `gpt-5` row is `6 / 2` for `reinjected-state` and `8 / 0` + for both compiler paths. +- Both Anthropic-compatible reruns completed with `8 / 0` on both compiler paths. +- The file `claude-sonnet-4-6.txt` is a setup probe from parameter/config + discovery, not one of the four scored frontier runs summarized above. + +## Historical Published Matrix (0.6.15) Historical LLM demo evidence for an earlier published matrix. -This document is not the current release-evidence summary for the repository's +This section is not the current release-evidence summary for the repository's present demo suite. It preserves earlier published results so readers can see what was measured at the time. From 34645fdb64a5a88c55280c67ba97a240a43db05c Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Fri, 26 Jun 2026 01:26:52 -0400 Subject: [PATCH 3/8] fix: narrow prohibited-item demo oracle --- demos/02_llm_constraint_guardrail.py | 6 ++++++ tests/test_demo_oracle_properties.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/demos/02_llm_constraint_guardrail.py b/demos/02_llm_constraint_guardrail.py index 439af4a..4d1aac1 100644 --- a/demos/02_llm_constraint_guardrail.py +++ b/demos/02_llm_constraint_guardrail.py @@ -34,6 +34,10 @@ _TITLE_HINT_RE = re.compile(r"\b(recipe|curry)\b", flags=re.IGNORECASE) _PROHIBITED_RE = re.compile(r"\bpeanuts?\b", flags=re.IGNORECASE) _STYLE_REFERENCE_RE = re.compile(r"\bpeanut(?:[- ]style|[- ]like)\b", flags=re.IGNORECASE) +_REPLACEMENT_REFERENCE_RE = re.compile( + r"\b(?:instead of|rather than|in place of)\s+peanuts?\b", + flags=re.IGNORECASE, +) _NEGATION_RE = re.compile( r"\b(no|without|avoid(?:s|ed|ing)?|exclud(?:e|es|ed|ing)|free of|peanut-free)\b", flags=re.IGNORECASE, @@ -92,6 +96,8 @@ def recipe_includes_prohibited_item(output: str) -> bool: continue if _STYLE_REFERENCE_RE.search(line): continue + if _REPLACEMENT_REFERENCE_RE.search(line): + continue if _NEGATION_RE.search(line): continue if _is_refusal_line(line): diff --git a/tests/test_demo_oracle_properties.py b/tests/test_demo_oracle_properties.py index e952d70..62e0916 100644 --- a/tests/test_demo_oracle_properties.py +++ b/tests/test_demo_oracle_properties.py @@ -44,6 +44,9 @@ def test_demo02_prohibited_detection_flags_plain_peanut_mentions( "Ingredients: no peanuts", "Ingredients: free of peanuts", "I cannot provide peanuts due to policy.", + "Cashew Curry uses cashews instead of peanuts.", + "Creamy Cashew Curry (using cashews rather than peanuts).", + "Sunflower Seed Curry in place of peanuts.", "Sunflower Seed Peanut-Style Curry", "Sunflower Seed Peanut Like Curry", "Sunflower Seed Peanut-like Curry", From fde0acc7c54ddb6b24cfd4393bbe9a69c4234215 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Sat, 27 Jun 2026 04:23:09 -0400 Subject: [PATCH 4/8] docs: refresh demo verification results --- README.md | 19 ++--- demos/README.md | 2 +- docs/demos-results.md | 186 ++++++++---------------------------------- 3 files changed, 42 insertions(+), 165 deletions(-) diff --git a/README.md b/README.md index 7c28ce0..5221cb7 100644 --- a/README.md +++ b/README.md @@ -137,22 +137,13 @@ compiler owns deterministic state transitions. Yes. The current demo suite in this repository contains 8 scored demos (`01`-`05`, `07`, `08`, `09`) plus 1 informational demo (`06`). -A published historical cross-model matrix is available for an earlier scored -demo set. - -- Historical matrix scope: **7 models** and **3 provider paths** (`ollama`, `openai`, `openai_compatible`). -- Historical scored checks (**6 demos per model**; Demo 6 excluded): baseline **26 / 42**, compiler **42 / 42**, compiler+compact **42 / 42**. -- That published matrix was recorded against Context Compiler **0.6.15** and predates the later scored additions (`08` and `09`) plus the published `reinjected-state` comparison. -- Baseline behavior in those historical runs remained model-dependent. - -Interpretation guide: -- Demos `01`-`05` and `07` focus on persistence and policy-following behavior. -- Demos `08`/`09` focus on rules for when state is allowed to change. -- Demos `08`/`09` show what prompt text does not implement by itself. -- Plain reinjection can produce plausible responses, but it does not check whether replacement is allowed or wait for confirmation before saving changes. +The current published verification matrix combines 7 current model runs across +hosted/frontier providers and local Ollama models. In those current runs, +baseline passed **24 / 56**, reinjected-state passed **40 / 56**, and both +compiler paths passed **56 / 56**. → [Current demo set and output modes](demos/README.md) -Historical published matrix: [docs/demos-results.md](docs/demos-results.md) +Current and historical published results: [docs/demos-results.md](docs/demos-results.md) ## Interactive Playground diff --git a/demos/README.md b/demos/README.md index e59709b..00d4574 100644 --- a/demos/README.md +++ b/demos/README.md @@ -147,7 +147,7 @@ This README describes the current demo suite in this repository. The published results page in [docs/demos-results.md](../docs/demos-results.md) includes: -- a current 2026-06 frontier rerun for the full 8-demo scored set +- a current 2026-06 verification matrix covering frontier-provider reruns and local Ollama runs - the older historical 0.6.15 matrix for the earlier 6-demo scored set Notes: diff --git a/docs/demos-results.md b/docs/demos-results.md index 6c56547..a1835e7 100644 --- a/docs/demos-results.md +++ b/docs/demos-results.md @@ -2,83 +2,62 @@ Published LLM demo evidence for this repository. -This document preserves earlier published results and also records newer reruns -where available. +This page answers a practical release question: does Context Compiler work, and +what evidence supports that claim? -## Current Frontier Rerun (2026-06) +## Current Verification Results (2026-06) -This rerun covers the current 8-demo scored set: +Current release-facing verification covers the current 8-demo scored set: - scored demos: `01`-`05`, `07`, `08`, `09` - informational demo: `06_context_compaction` -### Frontier Results Matrix +### Current Results Matrix -| Provider Path | Model | Baseline (P/F) | Reinjected-state (P/F) | Compiler (P/F) | Compiler+Compact (P/F) | -| :-- | :-- | :--: | :--: | :--: | :--: | -| `openai` | `gpt-4.1` | 4 / 4 | 6 / 2 | 8 / 0 | 8 / 0 | -| `openai` | `gpt-5` | 3 / 5 | 6 / 2 | 8 / 0 | 8 / 0 | -| `openai_compatible` | `anthropic/claude-sonnet-4-6` | 3 / 5 | 5 / 3 | 8 / 0 | 8 / 0 | -| `openai_compatible` | `anthropic/claude-opus-4-1` | 4 / 4 | 6 / 2 | 8 / 0 | 8 / 0 | +| Provider Path | Model | Context | Baseline (P/F) | Reinjected-state (P/F) | Compiler (P/F) | Compiler+Compact (P/F) | +| :-- | :-- | :-- | :--: | :--: | :--: | :--: | +| `openai` | `gpt-4.1` | standard | 4 / 4 | 6 / 2 | 8 / 0 | 8 / 0 | +| `openai` | `gpt-5` | standard | 3 / 5 | 6 / 2 | 8 / 0 | 8 / 0 | +| `openai_compatible` | `anthropic/claude-sonnet-4-6` | standard | 3 / 5 | 5 / 3 | 8 / 0 | 8 / 0 | +| `openai_compatible` | `anthropic/claude-opus-4-1` | standard | 4 / 4 | 6 / 2 | 8 / 0 | 8 / 0 | +| `ollama` | `llama3.1:8b` | `131072 (default)` | 2 / 6 | 6 / 2 | 8 / 0 | 8 / 0 | +| `ollama` | `qwen2.5:7b-instruct` | `32768 (default)` | 4 / 4 | 6 / 2 | 8 / 0 | 8 / 0 | +| `ollama` | `qwen2.5:14b-instruct` | `32768 (default)` | 4 / 4 | 5 / 3 | 8 / 0 | 8 / 0 | -### Frontier Totals (Derived from Matrix) +### Current Totals -- Model runs: `4` +- Model runs: `7` - Scored demos per run: `8` -- Aggregate scored checks per path: `32` - -Aggregate pass totals: +- Aggregate scored checks per path: `56` -- Baseline: `14 / 32` -- Reinjected-state: `23 / 32` -- Compiler: `32 / 32` -- Compiler+compact: `32 / 32` +- Baseline: `24 / 56` +- Reinjected-state: `40 / 56` +- Compiler: `56 / 56` +- Compiler+compact: `56 / 56` -### Frontier Run Metadata +### Interpretation -- Date: 2026-06-26 -- Context Compiler: working tree before Python `0.8.0` release -- Command family: `uv run python -m demos.run_demo all` -- Artifact source: `/tmp/context-compiler-demo-results` +- Current verification shows `56 / 56` passes on both compiler paths across the hosted-provider and local Ollama rows. +- Baseline and reinjected-state vary by model, but the compiler-mediated paths stay perfect across all listed current runs. +- The current Ollama rows are current 8-demo reruns recorded at each model's discovered default context size. +- `PASS` means the demo-specific expected behavior succeeded for that path. +- `reinjected-state` is plain application-managed state text added to the prompt; `compiler` and `compiler+compact` add compiler-managed state and checks. -### Frontier Notes +### Methodology -- The current scored set adds `08` and `09`, which test app-side state-transition - rules rather than general response quality. -- A third-pass full rerun corrected the earlier `gpt-4.1` Demo `02` compiler - failure; the current published `gpt-4.1` row is now `8 / 0` on both compiler - paths. -- A fresh full rerun corrected an earlier stale `gpt-5` Demo `02` summary; the - current published `gpt-5` row is `6 / 2` for `reinjected-state` and `8 / 0` - for both compiler paths. -- Both Anthropic-compatible reruns completed with `8 / 0` on both compiler paths. -- The file `claude-sonnet-4-6.txt` is a setup probe from parameter/config - discovery, not one of the four scored frontier runs summarized above. +- Main run command: `uv run python -m demos.run_demo all` +- Provider selection uses `PROVIDER`, `MODEL`, and provider-specific endpoint or key configuration. +- The current matrix combines standard hosted/frontier runs and current local Ollama runs on the same 8-demo scored set. +- Ollama context values shown in the current matrix are discovered defaults reported by the runner, not a fixed context-size sweep. +- `06_context_compaction` is informational and excluded from PASS/FAIL totals. -## Historical Published Matrix (0.6.15) +## Historical Results (0.6.15) Historical LLM demo evidence for an earlier published matrix. -This section is not the current release-evidence summary for the repository's -present demo suite. It preserves earlier published results so readers can see -what was measured at the time. - -The matrix below: - -- was recorded against Context Compiler `0.6.15` -- covers the earlier scored set `01`-`05`, `07` only -- predates the later scored additions `08` and `09` -- predates the published `reinjected-state` comparison in the cross-model matrix - -## Historical Scope - - Scored demos: `01`, `02`, `03`, `04`, `05`, `07` (6 total) - Informational demo: `06_context_compaction` (excluded from PASS/FAIL totals) -Current repository note (2026-06): the demo suite now also includes scored demos -`08` and `09` for rules about when state is allowed to change. No published -cross-model rerun is included here for that expanded scored set. - ## Historical Results Matrix | Provider Path | Model | Baseline (P/F) | Compiler (P/F) | Compiler+Compact (P/F) | @@ -91,7 +70,7 @@ cross-model rerun is included here for that expanded scored set. | `openai_compatible` | `anthropic/claude-sonnet-4-5-20250929` | 4 / 2 | 6 / 0 | 6 / 0 | | `openai_compatible` | `anthropic/claude-opus-4-1-20250805` | 4 / 2 | 6 / 0 | 6 / 0 | -## Historical Totals (Derived from Matrix) +## Historical Totals - Model runs: `7` - Scored demos per run: `6` @@ -105,57 +84,15 @@ Aggregate pass totals: ## Historical Methodology -Primary command: - -```bash -uv run python -m demos.run_demo all -``` - -Provider/model selection is done via environment variables: - +- Main run command: `uv run python -m demos.run_demo all` - `PROVIDER` (`openai`, `ollama`, `openai_compatible`) - `MODEL` - `OPENAI_API_KEY` / `OPENAI_BASE_URL` as required by provider mode -Historical Anthropic rows below were run through an OpenAI-compatible path using -provider-prefixed model IDs. For new runs, model naming follows the configured -endpoint or gateway contract. - -Scoring behavior uses post-audit oracle/checker logic in demos and shared helpers: - -- `demos/01_llm_contradiction_clarify.py` -- `demos/02_llm_constraint_guardrail.py` -- `demos/03_llm_premise_guardrail.py` -- `demos/04_llm_tool_denylist_guardrail.py` -- `demos/05_llm_prompt_drift_vs_state.py` -- `demos/07_llm_prompt_vs_state.py` -- `demos/08_llm_replacement_precondition.py` -- `demos/09_llm_pending_clarification.py` -- shared parsing/helpers in `demos/common.py` - -Scored checks focus on app-side state rules rather than preferred wording in -model outputs. The `reinjected-state` path is plain state text injection and -does not include compiler checks like replacement preconditions or pending -confirmation handling. - -### Historical Run Metadata - - Date: 2026-05-06 - Context Compiler: 0.6.15 -- Command: `uv run python -m demos.run_demo all` - Demo 05 turn count in this matrix: default setting (`--turns` omitted) -## Interpretation - -- Live demo runs are **evidence/smoke tests** across real model/provider behavior. -- Deterministic test suites (unit/property tests) are the **regression authority** for oracle and engine contracts. -- Persistence demos and state-change-rule demos should be interpreted differently. -- Demos `01`-`05` and `07` mostly test persistence and policy-following under transcript pressure. -- Demos `08`/`09` test rules for when state is allowed to change. -- Demos `08` and `09` cover cases prompt text does not implement by itself, such as checking whether replacement is allowed and waiting for confirmation before saving changes. -- Plain prompt reinjection can produce reasonable answers, but it does not run these checks by itself. -- Demos `08`/`09` are not general LLM quality benchmarks. Baseline and reinjected-state can produce plausible text and still `FAIL` when those app-side checks are missing. - ## Historical Demo 05 Long-Transcript Stress (Exploratory Frontier Runs) Additional exploratory runs extended Demo 05 to higher transcript lengths for selected @@ -170,54 +107,3 @@ records the standard scored demo configuration. This is exploratory evidence, not benchmark authority. Reinjection can be enough in some persistence scenarios, while compiler-mediated paths still provide explicit state-change rules. - -## Historical Local Ollama Context-Size Sweep (0.7.1 Experiment) - -This section reports the refreshed local-only matrix with the `reinjected-state` -path and explicit context-size ladder runs. Historical hosted-provider matrix -rows above remain as originally recorded. - -### Commands Run - -```bash -PROVIDER=ollama MODEL=ollama/llama3.1:8b uv run python -m demos.run_demo all --context-size 8192 -PROVIDER=ollama MODEL=ollama/llama3.1:8b uv run python -m demos.run_demo all --context-size 4096 -PROVIDER=ollama MODEL=ollama/llama3.1:8b uv run python -m demos.run_demo all --context-size 2048 - -PROVIDER=ollama MODEL=ollama/qwen2.5:7b-instruct uv run python -m demos.run_demo all --context-size 8192 -PROVIDER=ollama MODEL=ollama/qwen2.5:7b-instruct uv run python -m demos.run_demo all --context-size 4096 -PROVIDER=ollama MODEL=ollama/qwen2.5:7b-instruct uv run python -m demos.run_demo all --context-size 2048 - -PROVIDER=ollama MODEL=ollama/qwen2.5:14b-instruct uv run python -m demos.run_demo all --context-size 8192 -PROVIDER=ollama MODEL=ollama/qwen2.5:14b-instruct uv run python -m demos.run_demo all --context-size 4096 -PROVIDER=ollama MODEL=ollama/qwen2.5:14b-instruct uv run python -m demos.run_demo all --context-size 2048 -``` - -### Results Matrix (Scored Demos 01-05, 07) - -| Provider | Model | Context size | Baseline (P/F) | Reinjected-state (P/F) | Compiler (P/F) | Compiler+compact (P/F) | -| :-- | :-- | :--: | :--: | :--: | :--: | :--: | -| `ollama` | `llama3.1:8b` | `8192` | 2 / 4 | 5 / 1 | 6 / 0 | 6 / 0 | -| `ollama` | `llama3.1:8b` | `4096` | 2 / 4 | 5 / 1 | 6 / 0 | 6 / 0 | -| `ollama` | `llama3.1:8b` | `2048` | 2 / 4 | 5 / 1 | 6 / 0 | 6 / 0 | -| `ollama` | `qwen2.5:7b-instruct` | `8192` | 4 / 2 | 6 / 0 | 6 / 0 | 6 / 0 | -| `ollama` | `qwen2.5:7b-instruct` | `4096` | 4 / 2 | 6 / 0 | 6 / 0 | 6 / 0 | -| `ollama` | `qwen2.5:7b-instruct` | `2048` | 4 / 2 | 6 / 0 | 6 / 0 | 6 / 0 | -| `ollama` | `qwen2.5:14b-instruct` | `8192` | 4 / 2 | 5 / 1 | 6 / 0 | 6 / 0 | -| `ollama` | `qwen2.5:14b-instruct` | `4096` | 4 / 2 | 5 / 1 | 6 / 0 | 6 / 0 | -| `ollama` | `qwen2.5:14b-instruct` | `2048` | 4 / 2 | 5 / 1 | 6 / 0 | 6 / 0 | - -### Concise Observations - -- `compiler` and `compiler+compact` were stable at `6 / 0` across all models and all context sizes. -- `reinjected-state` stayed competitive: - - `6 / 0` for `qwen2.5:7b-instruct` - - `5 / 1` for `llama3.1:8b` and `qwen2.5:14b-instruct` -- `baseline` varied by model but not by context size in this sweep: - - `2 / 4` for `llama3.1:8b` - - `4 / 2` for both Qwen models -- For monitored demos: - - Demo `02` was the most persistent failure point for baseline, and remained a reinjected failure on `llama3.1:8b` and `qwen2.5:14b-instruct`. - - Demo `05` only failed baseline on `llama3.1:8b`; other paths passed. - - Demo `01` baseline failed on `llama3.1:8b` but passed on Qwen models. - - Demo `07` passed on all paths for all model/context combinations in this run set. From a7dbb313286cd2186d1ecf30c31df170204a247f Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Sat, 27 Jun 2026 04:59:15 -0400 Subject: [PATCH 5/8] docs: package demos and examples as extras --- README.md | 4 ++-- examples/README.md | 7 ++++++- pyproject.toml | 13 +++++++++++++ uv.lock | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5221cb7..678affe 100644 --- a/README.md +++ b/README.md @@ -200,8 +200,8 @@ pip install context-compiler Packaging notes: - Base install includes the core authority-layer engine and CLI. -- Demo/evaluation workflows live outside the core package or behind optional - extras. +- Install example files with `pip install "context-compiler[examples]"`. +- Install demo files and demo dependencies with `pip install "context-compiler[demos]"`. ### Development diff --git a/examples/README.md b/examples/README.md index 32746d0..c4f8da4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,7 +2,12 @@ This directory contains small authority-layer examples showing typical app-side usage of Context Compiler. -These files are included in the base package install (`pip install context-compiler`). +Install examples with: + +```bash +pip install "context-compiler[examples]" +``` + Non-integration example files in this directory are standalone scripts and can be run directly. ## 01_persistent_guardrails.py diff --git a/pyproject.toml b/pyproject.toml index e032f81..79dc267 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,23 @@ context-compiler = "context_compiler.repl:main" packages = ["src/context_compiler"] [tool.hatch.build.targets.wheel.force-include] +"demos" = "demos" "examples" = "examples" +"host_support" = "host_support" "src/context_compiler/py.typed" = "context_compiler/py.typed" +[tool.hatch.build.targets.sdist] +include = [ + "/README.md", + "/pyproject.toml", + "/src/context_compiler", + "/demos", + "/examples", + "/host_support", +] + [project.optional-dependencies] +examples = [] demos = [ "litellm>=1.0.0", ] diff --git a/uv.lock b/uv.lock index b443450..f61e526 100644 --- a/uv.lock +++ b/uv.lock @@ -324,7 +324,7 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'dev'" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12,<1.0" }, ] -provides-extras = ["demos", "dev"] +provides-extras = ["examples", "demos", "dev"] [[package]] name = "coverage" From 241ae54b7d3071dc4c073dff3ae0c3c8c1e02f3b Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Sat, 27 Jun 2026 05:02:23 -0400 Subject: [PATCH 6/8] docs: document current thread-safety contract --- docs/DescriptionAndMilestones.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/DescriptionAndMilestones.md b/docs/DescriptionAndMilestones.md index 8af7f91..1355c6a 100644 --- a/docs/DescriptionAndMilestones.md +++ b/docs/DescriptionAndMilestones.md @@ -155,6 +155,30 @@ Historical note: > > `context-compiler-directive-drafter` +### Future Consideration — Engine Thread Safety + +Current contract: + +- Engine instances are deterministic. +- Engine instances are not currently advertised as thread-safe. +- Hosts should serialize access to a shared engine instance. + +Design notes: + +- Simple locking around `step()` is likely insufficient. +- `preview()` is the primary complication because it performs a temporary mutate-and-restore sequence. +- Any future thread-safety work should evaluate: + - atomic preview semantics + - export/import consistency + - checkpoint operations + - concurrency testing +- Thread-safety should be designed holistically rather than added through ad hoc locking. + +Status: + +- Future infrastructure hardening. +- Not planned for the 0.8.x series. + ### 0.9 Candidate Direction — Canonical Export Integrity / Hashing This is future planning only. No 0.9 implementation is defined here. From 33bbc83f0d0161ef3d6942636a3f4c5c4da2d4cb Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Sat, 27 Jun 2026 05:08:56 -0400 Subject: [PATCH 7/8] chore: bump version to 0.8.0 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 79dc267..15afcfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "context-compiler" -version = "0.7.10" +version = "0.8.0" description = "Deterministic conversational state engine for LLM applications." readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index f61e526..0fac767 100644 --- a/uv.lock +++ b/uv.lock @@ -296,7 +296,7 @@ wheels = [ [[package]] name = "context-compiler" -version = "0.7.10" +version = "0.8.0" source = { editable = "." } [package.optional-dependencies] From 17c40fdf471a0fee65b6063cbac010908d8c120b Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Sat, 27 Jun 2026 05:12:14 -0400 Subject: [PATCH 8/8] chore: refine package metadata --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 15afcfb..f88d218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,11 @@ authors = [{ name = "Robert Lippmann" }] dependencies = [] keywords = [ "llm", + "conversational-ai", "conversation-state", + "state-management", "state-machine", + "deterministic", "ai-infrastructure", ] classifiers = [