From 8a31ecd6179bd77b0cbb7de96146dd8919d8b195 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Fri, 19 Jun 2026 16:55:12 -0400 Subject: [PATCH 1/2] feat: add JSONL range filters and field snippets --- .plans/jsonl-field-search-snippets.md | 228 +++++++++++ .plans/jsonl-search-range-filters.md | 219 +++++++++++ README.md | 15 +- docs/behavior.md | 28 ++ docs/docs.md | 55 +++ docs/site/about/index.html | 6 +- docs/site/explainer/index.html | 6 +- docs/site/index.html | 4 +- docs/table.md | 2 +- tests/test_file_tools.py | 533 ++++++++++++++++++++++++++ thinharness/defaults.py | 4 +- thinharness/tools/__init__.py | 3 +- thinharness/tools/jsonl.py | 425 ++++++++++++++++++-- 13 files changed, 1475 insertions(+), 53 deletions(-) create mode 100644 .plans/jsonl-field-search-snippets.md create mode 100644 .plans/jsonl-search-range-filters.md diff --git a/.plans/jsonl-field-search-snippets.md b/.plans/jsonl-field-search-snippets.md new file mode 100644 index 0000000..f936178 --- /dev/null +++ b/.plans/jsonl-field-search-snippets.md @@ -0,0 +1,228 @@ +# JSONL Field Search Snippets Plan + +## Goal + +Add a compact field-level search mode to `jsonl_search` so callers can select JSONL rows with the existing query/where machinery, then search inside one or more large string fields and return only matching internal lines/snippets. + +This is meant for structured JSONL rows that contain large multiline strings such as accessibility trees, browser state dumps, transcript blobs, logs, or rendered document text. The caller often does not need the whole field; they need the few lines inside the field that match a term or regex. + +## Problem + +Current `jsonl_search` has grep capabilities at the row-selection stage: + +1. Optional ripgrep query finds candidate JSONL lines/rows. +2. Python parses candidate rows. +3. `where` filters keep or reject rows. +4. `fields` projects whole values from each kept row. + +That works when projected fields are small. It is weak when one projected field is a huge multiline string. A row may match, but returning `accessibility_tree` or `text` still produces one large blob. If the caller sets a per-field cap, the important lines can be clipped. If the caller sets `0` for no per-field truncation, the whole tool output can still hit the global tool cap and spill to a file. + +The missing behavior is not more row filtering. It is an output transformation: after a row is selected, search inside a selected string field, split it into lines, and return only the matching lines plus optional nearby context. + +## Proposed API Shape + +Add `field_searches` to `JsonlSearchArgs`: + +```python +class JsonlFieldSearch(StrictArgs): + field: str + query: str + regex: bool = False + case_sensitive: bool = False + context_lines: int = 0 + max_matches: int = 20 + max_line_chars: int = 300 + +class JsonlSearchArgs(StrictArgs): + query: str = "" + path: str = "." + fields: dict[str, int] = Field(default_factory=dict) + where: list[JsonlWhereFilter] = Field(default_factory=list) + field_searches: list[JsonlFieldSearch] = Field(default_factory=list) + max_files: int = 100 + max_matches_per_file: int = 25 + timeout: int | None = None + max_chars: int | None = None +``` + +Example call: + +```json +{ + "path": "trajectories/1d56a4d6/states.jsonl", + "where": [{"field": "state_index", "op": "eq", "value": "11"}], + "fields": {"state_index": 0, "url": 0}, + "field_searches": [ + { + "field": "accessibility_tree", + "query": "Incident|-- None --|Edit personal filters", + "regex": true, + "context_lines": 0, + "max_matches": 20, + "max_line_chars": 300 + } + ] +} +``` + +Example output: + +```text +summary: + query: (none) + scope: path=trajectories/1d56a4d6/states.jsonl + where: state_index eq '11' + fields: state_index, url + field_searches: accessibility_tree + files: 1 total, 1 shown + rows_matched: 1 + +trajectories/1d56a4d6/states.jsonl + 12: {"state_index": 11, "url": "https://..."} + accessibility_tree matches: + 426: [a800] menuitem 'Edit personal filters', visible + 428: [a802] menuitem '-- None --', visible + 433: [a807] menuitem 'Incident Mobile', visible + 434: [a808] menuitem 'Incident Portal', visible + 435: [a809] menuitem 'My Open Incidents', visible +``` + +## Semantics + +`query` keeps its current meaning: optional ripgrep prefilter over JSONL rows. `where` keeps its current meaning: structured row filtering after JSON parsing. `fields` keeps its current meaning: normal projected row values. + +`field_searches` runs only after a row has already passed `query` and `where`. It does not decide candidate rows by itself unless `query` is omitted and `where` selects rows. Each field search: + +- Resolves `field` with the same jq-style field path parser used by `fields` and `where`. +- Requires the resolved field value to be a string. Missing, null, object, array, number, and boolean values produce a compact note and no matches for that field. +- Splits the string with `splitlines()`. +- Matches each internal field line with either substring search or regex search. +- Includes `context_lines` lines before and after each match, merging overlapping ranges. +- Returns at most `max_matches` primary matches per field search per row. Context lines do not count as primary matches. +- Truncates each returned internal line to `max_line_chars`. + +`context_lines` is the same idea as `grep -C`: `0` returns only matching lines, `1` returns one neighboring line before and after each match, and so on. + +## Multiple Field Searches + +Support multiple searches per row. This lets callers inspect more than one large field without issuing multiple tool calls: + +```json +{ + "fields": {"state_index": 0}, + "field_searches": [ + {"field": "accessibility_tree", "query": "Incident", "regex": false}, + {"field": "thought", "query": "Filters", "regex": false, "context_lines": 1} + ] +} +``` + +Output should group snippets under the field name: + +```text + 12: {"state_index": 11} + accessibility_tree matches: + ... + thought matches: + ... +``` + +If the same field appears multiple times in `field_searches`, preserve call order and include a short query label so the outputs are distinguishable. + +## Validation Rules + +- `field` must be a non-empty string. +- `query` must be a non-empty string. +- `context_lines` must be `>= 0`. +- `max_matches` must be `>= 1`. +- `max_line_chars` must be `>= 1`. +- Invalid regex should fail the search with `invalid field_search regex`. +- Invalid field paths should fail the search with `invalid field path`, matching existing projection behavior. + +## Output Details + +If no internal field lines match for a selected row, show a compact zero-match line only when no normal fields were projected and all field searches missed. Avoid noisy no-match blocks when normal projected fields already make the row useful. + +Suggested zero-match output: + +```text + 12: {} + accessibility_tree matches: none +``` + +When matches are omitted because `max_matches` was reached, include: + +```text + ... 14 more match(es) +``` + +This omitted count should count additional primary matching lines, not context lines. + +The global spill/truncation behavior should remain unchanged. The new mode is intended to reduce how often useful output spills by returning snippets instead of full large fields. + +## Implementation Sketch + +1. Add `JsonlFieldSearch` and `field_searches` to `thinharness/tools/jsonl.py`. +2. Compile field searches before scanning rows: + - parse field path once + - compile regex once when `regex=true` + - normalize case behavior +3. Extend the row formatting path in `JsonlSearch.search()`: + - keep existing `_project_fields(row, fields)` behavior for `fields` + - compute `field_search` snippet blocks for each shown row + - render projected fields and snippet blocks together under the row line +4. Add helpers: + - `_compile_field_searches(...)` + - `_field_search_matches(row, compiled_searches) -> list[RenderedFieldSearch]` + - `_line_ranges_for_matches(...)` to merge context windows + - `_truncate_internal_line(...)` +5. Update the summary header to list searched fields when `field_searches` is non-empty. +6. Update `DEFAULT_JSONL_SEARCH_INSTRUCTIONS` with a concise note that `field_searches` is for extracting matching lines from large string fields. +7. Add focused tests in `tests/test_file_tools.py`. + +## Test Plan + +Add tests for: + +- Field search returns matching internal lines from a large multiline string. +- Field search works when `query` is omitted and `where` selects the row. +- Top-level `query` still acts only as row prefilter. +- Normal `fields` projection and `field_searches` render together. +- Multiple field searches render in order. +- Duplicate field searches are distinguishable. +- `context_lines` includes neighboring lines. +- Overlapping context ranges merge without duplicate output. +- `max_matches` limits primary matches and reports omitted match count. +- `max_line_chars` truncates returned internal lines. +- Substring search is case-insensitive by default. +- `case_sensitive=true` changes substring matching. +- Regex search works. +- Invalid regex fails the search with a clear error. +- Missing/non-string field values do not crash and produce compact no-match behavior. +- Invalid field path fails consistently with existing field projection errors. +- Existing `jsonl_search` behavior without `field_searches` is unchanged. +- Tool schema exposes `field_searches` and its validation constraints. + +Validation commands: + +```bash +uv run pytest tests/test_file_tools.py -k jsonl_search +uv run pytest tests/test_file_tools.py +uv run ruff check thinharness/tools/jsonl.py thinharness/defaults.py tests/test_file_tools.py +uv run pyright +``` + +## Non-Goals + +- Do not add a benchmark-specific trajectory inspection tool. +- Do not remove global tool output truncation or spill-to-file behavior. +- Do not change the meaning of top-level `query`. +- Do not make `fields` do two jobs; normal projection stays in `fields`, snippet extraction goes in `field_searches`. +- Do not add OR filters, sorting, scoring, or ranking. +- Do not add a dependency for regex or text extraction. + +## Open Checks During Implementation + +- Confirm whether the model schema remains understandable with nested `field_searches`; if needed, add examples to the default tool instructions rather than broad docs rewrites. +- Decide whether no-match snippet blocks should always be shown or only when the row would otherwise be empty. Prefer compact output unless tests show ambiguity. +- Confirm line numbering should be 1-based within the field string, not source-file line numbers. The row already has the JSONL source line number. diff --git a/.plans/jsonl-search-range-filters.md b/.plans/jsonl-search-range-filters.md new file mode 100644 index 0000000..0c0a536 --- /dev/null +++ b/.plans/jsonl-search-range-filters.md @@ -0,0 +1,219 @@ +# JSONL Search Range Filters Plan + +## Goal + +Add `gt`, `gte`, `lt`, and `lte` operators to `jsonl_search` so callers can filter JSONL rows by numeric and date-like scalar fields. + +The feature should remain small, predictable, and compatible with the current `where` shape: + +```json +{"field": "score", "op": "gte", "value": "0.8", "type": "number"} +{"field": "published_at", "op": "lt", "value": "2026-06-12", "type": "date"} +``` + +## Current Behavior + +`jsonl_search` is implemented in `thinharness/tools/jsonl.py`. + +Current `where` operators: + +- `eq` +- `ne` +- `in` +- `contains` +- `regex` +- `exists` + +Current comparison behavior renders non-string JSON values to strings for most operators. That is useful for text filters but too ambiguous for range filters. + +## Decisions + +1. Add the new operators to the existing `where` API. +2. Require an explicit `type` for range operators. +3. Support `type: "number"` and `type: "date"` initially. +4. For `type: "number"`, only real JSON numbers in rows are comparable. Numeric strings in row data do not match. +5. For `type: "date"`, support ISO-ish strings only. Do not support Unix timestamps as date filters in this feature. +6. Date-only filters and date-only row values compare by calendar date. Datetime row values compared to date-only filter values are compared by their calendar date, so `2026-06-12T15:30:00Z <= 2026-06-12` is true. +7. Rows with missing, null, object, array, invalid, or wrong-type comparison values do not match. +8. Search output should warn when at least one row could not be compared for a range filter. +9. Invalid filter definitions should still fail the whole search. +10. Range filters are pre-validated before scanning rows, so invalid filters fail even when the scope has zero candidate rows. + +## Proposed API Shape + +Update `JsonlWhereFilter`: + +```python +class JsonlWhereFilter(StrictArgs): + field: str + op: Literal["eq", "ne", "in", "contains", "regex", "exists", "gt", "gte", "lt", "lte"] + value: str | None = None + values: list[str] | None = None + type: Literal["number", "date"] | None = None +``` + +Rules: + +- `gt`, `gte`, `lt`, and `lte` require `value`. +- `gt`, `gte`, `lt`, and `lte` require `type`. +- Existing non-range operators reject `type` if supplied. +- Range operators reject `values` if supplied. +- Keep `value` as `str | None` for now. The filter target is parsed according to `type`. +- Empty string range values are invalid filter definitions and fail the search. + +## Number Semantics + +For `type: "number"`: + +- Filter target must parse as a finite number. Parsing with `float()` is not enough; explicitly reject `NaN`, `Infinity`, and `-Infinity` with `math.isfinite()`. +- Row value must be an `int` or `float`, excluding booleans. +- `NaN`, `Infinity`, and `-Infinity` should not be treated as comparable if encountered. +- Numeric strings in rows do not match. + +Examples: + +```json +{"score": 9.5} +``` + +passes: + +```json +{"field": "score", "op": "gt", "value": "8", "type": "number"} +``` + +but: + +```json +{"score": "9.5"} +``` + +does not pass the same filter. + +## Date Semantics + +For `type: "date"`: + +- Filter target must be an ISO-ish date or datetime string. +- Row value must be a string parseable as an ISO-ish date or datetime. +- Supported examples: + - `2026-06-12` + - `2026-06-12T14:03:00` + - `2026-06-12T14:03:00Z` + - `2026-06-12T14:03:00-04:00` +- Use the Python standard library if possible, likely `datetime.date.fromisoformat()` and `datetime.datetime.fromisoformat()` with a small `Z` to `+00:00` normalization. +- Do not add a dependency just for date parsing unless the standard library approach proves too brittle. + +Comparison normalization: + +- If either side is date-only, reduce both sides to calendar dates before comparing. Strict operators remain strict calendar-date comparisons, so `2026-06-12T15:30:00Z > 2026-06-12` is false and `2026-06-12T15:30:00Z >= 2026-06-12` is true. +- For timezone-aware datetimes reduced to dates, use the date as written in the row string, not a UTC-normalized date. For example, `2026-06-12T23:30:00-04:00` compares as calendar date `2026-06-12` against a date-only filter. +- If both sides are datetimes, compare datetimes. +- If both datetimes are timezone-aware, compare by instant. +- If one datetime is timezone-aware and the other is naive, treat the row as not comparable rather than guessing a timezone. + +This avoids surprising date-only behavior. A filter like `lte 2026-06-12` naturally includes rows from any time on `2026-06-12`. + +## Warning Behavior + +Rows that cannot be compared should not fail the whole search. They should be counted and treated as non-matching. + +Warning count semantics: + +- `compare_warnings` is a count of distinct candidate rows where a range comparison was actually attempted and failed because the row value was non-comparable. +- A row is counted at most once even if multiple range filters on that row are non-comparable. +- Rows filtered out by the ripgrep prefilter are not candidate rows and are not counted. +- Rows that fail an earlier `where` filter before a later range filter is attempted are not counted for the later range filter. This keeps the current AND short-circuit behavior. +- Missing, null, object, array, wrong-type, non-finite number, invalid date string, and aware/naive datetime mismatch values count as non-comparable when the relevant range comparison is attempted. + +The result header should include a warning only when the count is nonzero, for example: + +```text +summary: + query: (none) + scope: path=events.jsonl + where: score gte '0.8' (number) + files: 1 total, 1 shown + rows_matched: 3 + compare_warnings: 2 row(s) had non-comparable values +``` + +Metadata should also expose the warning count for programmatic callers: + +```json +{"compare_warnings": 2} +``` + +Do not use the generic metadata key `warning` for comparison warnings. Ripgrep partial results already use `warning` and `warning_excerpt`; comparison warnings must use `compare_warnings` so both can coexist. + +## Implementation Sketch + +1. Extend `JsonlWhereFilter.op` and add `type`. +2. Add a compiled/pre-validated representation for `where` filters before candidate iteration starts: + - validate required and forbidden fields + - parse range filter targets exactly once + - fail the search with `invalid where filter` before reading rows when filter definitions are invalid +3. Add helper types/functions in `thinharness/tools/jsonl.py`: + - parse row scalar values by requested comparison type + - evaluate `gt/gte/lt/lte` + - return pass/fail plus whether the attempted range comparison was non-comparable, likely via a small result tuple or enum rather than a bare bool +4. Keep existing string operator behavior unchanged. +5. Update `_describe_where()` to show range filter types only for range filters. Existing operator display should not change. +6. Update `JsonlSearch.search()` to include warning counts in output and metadata. +7. Update `DEFAULT_JSONL_SEARCH_INSTRUCTIONS` with one concise bullet for range filters. +8. Add focused tests in `tests/test_file_tools.py`. + +## Test Plan + +Add tests for: + +- `number` range filters match JSON numbers. +- `gt` and `lt` are exclusive for equal numeric values. +- Numeric strings in row values do not match and increment warning count. +- Booleans do not count as numbers. +- Negative filter targets parse correctly. +- Non-finite row numbers such as `NaN`, `Infinity`, and `-Infinity` do not match and increment warning count if parsed by `json.loads`. +- Non-finite filter targets such as `NaN`, `Infinity`, and `-Infinity` fail as invalid filters. +- `date` filters match ISO date strings. +- Datetime-to-datetime filters work when both sides are naive or both sides are aware. +- Date-only filters include datetimes on the same calendar day for `lte` and `gte`. +- Date-only filters exclude datetimes on the same calendar day for strict `lt` and `gt`. +- Near-midnight timezone-offset datetimes compared to date-only filters use the date as written, not UTC-normalized date. +- Aware-vs-naive datetime comparison does not match and increments warning count. +- Invalid row date strings do not match and increment warning count. +- Missing/null/object/array fields do not match and increment warning count for range filters. +- Invalid range filter targets fail with `invalid where filter`. +- Invalid range filter targets fail even when there are zero candidate rows. +- Missing `type` for range ops fails. +- `type` supplied with a non-range op fails. +- `values` supplied with a range op fails. +- `compare_warnings` is absent from header and metadata when zero. +- `compare_warnings` coexists with ripgrep partial-warning metadata without overwriting `warning`. +- Multiple range filters count each candidate row at most once. +- A row that fails an earlier filter before a later range comparison is not counted for that later comparison. +- The tool schema exposes `gt`, `gte`, `lt`, `lte`, and `type: "number" | "date"` in `where` filters. +- `_describe_where()` includes `(number)` or `(date)` only for range filters. +- Existing `eq`, `regex`, `contains`, `in`, and `exists` behavior remains unchanged. + +Validation commands after implementation: + +```bash +uv run pytest tests/test_file_tools.py -k jsonl_search +uv run pytest tests/test_file_tools.py +uv run ruff check thinharness/tools/jsonl.py tests/test_file_tools.py +uv run pyright +``` + +## Non-Goals + +- No OR filters. +- No sorting. +- No automatic type inference. +- No Unix timestamp date mode. +- No dependency addition unless standard-library ISO parsing is insufficient. +- No broad docs rewrite beyond the tool instructions and any narrow README/docs mention already covering `jsonl_search`. + +## Open Checks During Implementation + +- Confirm pydantic accepts the field name `type` cleanly in `StrictArgs`; expected outcome is that a bare `type` field works. If it does not, use an internal alias while preserving the external JSON field name. +- Confirm schema generation includes `type` clearly enough for model tool calls. Add a schema assertion if needed. diff --git a/README.md b/README.md index 9e5760d..58d61e4 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,10 @@ I started building ThinHarness after running into this gap in practice. Filesyst domain-specific modalities (voice/realtime), eval/optimizer suites, UI/CLI tools, A2A/declarative wire protocols, code-executor backends. Provider implementations stay IN (they're part of what you import to use the library). - The exact tokei command + upstream commit hash for each row is in an HTML - comment above the row, so the number is reproducible. Measured 2026-06-15 - against the commit pinned in each row's comment. + The exact tokei command + upstream commit hash for each upstream row is in an + HTML comment above the row, so the number is reproducible. Upstream rows were + measured 2026-06-15 against the pinned commits; ThinHarness was remeasured + from this working tree on 2026-06-19. -->
@@ -55,10 +56,10 @@ I started building ThinHarness after running into this gap in practice. Filesyst - + ThinHarness - 7,892 + 8,197 ✅ ✅ ✅ @@ -227,7 +228,7 @@ ThinHarness has opinions. They are the reason it stays small. **Skills are tools, not auto-discovery.** Skills live in directories you point at explicitly. The agent calls `skill_read` and `skill_run` like any other tool. No interactive scan of the workspace, no global skill marketplace, no magic. SDK use is deliberate; the auto-discovery design is for interactive coding agents and doesn't belong here. -**Search is a top priority.** The `search` tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a `jsonl_search` variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data (line-delimited, naturally chunked, `jq` + `rg`). +**Search is a top priority.** The `search` tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a `jsonl_search` variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, `where` filters, range filters, and snippets from large multiline fields. **Parallel LLM calls, built in.** Fan out from inside the harness when a workflow needs reliability beyond a single agent loop — majority vote, ensembled extraction. Set `builtin_parallel_llm_model` to enable the default `parallel_llm` tool for plain-text batches; for validated structured output per call, instantiate `ParallelLlmTool` yourself with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. @@ -282,7 +283,7 @@ Streaming emits coarse run, model, tool, background, retry, limit, and subagent ## Features - **Filesystem tools:** `read`, `write`, batched exact-replacement `edit`, `search`, `list`, and `glob` with root-scoped path policies. -- **JSONL search:** opt-in `jsonl_search` for structured line-delimited data, with ripgrep prefiltering, field projection, and `where` filters. +- **JSONL search:** opt-in `jsonl_search` for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range `where` filters, and field-level snippets from large multiline string values. - **Bash prototype tool:** opt-in `BashTool` for exploratory shell commands. It is lightweight, custom-registration only, and is not included in the default or built-in tool set. - **Provider adapters:** built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider. - **Custom typed tools:** define sync or async `ToolSpec` handlers with Pydantic argument models, normalized `ToolResult` envelopes, sequential/background/approval flags, and per-tool retry settings. diff --git a/docs/behavior.md b/docs/behavior.md index 3305ea6..54ecd95 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -20,3 +20,31 @@ Use an uppercase, readable requirement prefix from the section name, such as `BA Use this section only when ordering, lifecycle, concurrency, retries, streaming, cancellation, or multi-actor behavior matters. --> + +## JSONL Field Search Snippets + +### Purpose + +`jsonl_search` can extract matching internal lines from large multiline string fields after a JSONL row has been selected by the existing row query and `where` filters. + +### Requirements + +- JSONL-FIELD-SEARCH-1: The top-level `query` remains a row prefilter over JSONL lines; `field_searches` runs only after JSON parsing and `where` filtering. +- JSONL-FIELD-SEARCH-2: Each field search resolves the same jq-style field paths used by `fields` and `where`, requires a non-empty query, and searches only string field values. +- JSONL-FIELD-SEARCH-3: Field searches support substring or regex matching, case-insensitive matching by default, optional case-sensitive matching, context lines, per-row match limits, and per-line truncation. +- JSONL-FIELD-SEARCH-4: Output preserves normal `fields` projection and renders matching field snippets beneath each selected row, without changing the existing global tool truncation and spill behavior. + +## JSONL Search Range Filters + +### Purpose + +`jsonl_search` can filter rows by numeric and date-like scalar fields using explicit range operators in the existing `where` filter shape. + +### Requirements + +- JSONL-RANGE-1: Range filters use `gt`, `gte`, `lt`, and `lte` operators and require an explicit `type` of `number` or `date`. +- JSONL-RANGE-2: Number range filters compare only JSON number values, excluding booleans and non-finite numbers; numeric strings do not match. +- JSONL-RANGE-3: Date range filters compare ISO-like date and datetime strings, compare date-only values by calendar date, and treat aware/naive datetime mismatches as non-comparable. +- JSONL-RANGE-4: Invalid range filter definitions fail before scanning rows with `invalid where filter`. +- JSONL-RANGE-5: Non-comparable row values do not match and increment `compare_warnings` once per candidate row where a range comparison was attempted. +- JSONL-RANGE-6: Comparison warnings appear in result metadata under `compare_warnings` without replacing ripgrep partial-result warning metadata. diff --git a/docs/docs.md b/docs/docs.md index 03663d3..c6ba132 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -124,6 +124,61 @@ harness = Harness(HarnessConfig( )) ``` +Use `query` as a ripgrep row prefilter, `fields` to project only the values the model needs, and `where` for structured filters over jq-style field paths: + +```python +result = await harness.run( + "Use jsonl_search on support/events.jsonl. Find open tickets with priority p1 and return id, customer.name, and updated_at." +) +``` + +The model can call `jsonl_search` with arguments like: + +```json +{ + "path": "support/events.jsonl", + "query": "ticket", + "where": [ + {"field": "status", "op": "eq", "value": "open"}, + {"field": "priority", "op": "eq", "value": "p1"} + ], + "fields": {"id": 0, "customer.name": 0, "updated_at": 0} +} +``` + +Range filters use `op` values `gt`, `gte`, `lt`, or `lte` with an explicit `type` of `number` or `date`. Number ranges match JSON numbers only; date ranges compare ISO-like date or datetime strings: + +```json +{ + "path": "support/events.jsonl", + "where": [ + {"field": "score", "op": "gte", "value": "0.8", "type": "number"}, + {"field": "created_at", "op": "gte", "value": "2026-06-01", "type": "date"} + ], + "fields": {"id": 0, "score": 0, "created_at": 0} +} +``` + +For large multiline string fields, `field_searches` returns matching internal lines without rendering the whole field. It runs after JSON parsing and `where` filtering, so it is best used with `fields` for the row summary and snippets for the bulky field: + +```json +{ + "path": "states.jsonl", + "where": [{"field": "state_index", "op": "eq", "value": "11"}], + "fields": {"state_index": 0, "url": 0}, + "field_searches": [ + { + "field": "accessibility_tree", + "query": "Incident|-- None --|Edit personal filters", + "regex": true, + "context_lines": 1, + "max_matches": 5, + "max_line_chars": 160 + } + ] +} +``` + Filesystem tools enforce the configured read and write policies. Paths must resolve under `root`; escape attempts through absolute paths outside `root`, `..`, or symlinks are rejected. ```python diff --git a/docs/site/about/index.html b/docs/site/about/index.html index 25b0a40..d6c8b04 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -78,7 +78,7 @@

How small, exactly

ThinHarness
- 7,892 + 8,197 ●●●●●● @@ -147,7 +147,7 @@

Opinions

purpose_built

Purpose-built agents, not universal agents

ThinHarness is for bounded agent loops inside software you control, not open-ended interactive assistants. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority.

no_bash

No bash by default

Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness keeps bash out of the default and built-in tool sets, but exposes an opt-in BashTool for exploratory runs before the workflow is hardened with typed tools.

skills

Skills are tools, not auto-discovery

Skills live in directories you point at explicitly. The agent calls skill_read and skill_run like any other tool. No interactive scan of the workspace, no global skill marketplace, no magic. SDK use is deliberate; the auto-discovery design is for interactive coding agents and doesn't belong here.

-
search

Search is a top priority

The search tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a jsonl_search variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data (line-delimited, naturally chunked, jq + rg).

+
search

Search is a top priority

The search tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a jsonl_search variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, where filters, range filters, and snippets from large multiline fields.

parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs reliability beyond a single agent loop — majority vote, ensembled extraction. Set builtin_parallel_llm_model to enable the default parallel_llm tool for plain-text batches; for validated structured output per call, instantiate ParallelLlmTool yourself with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

background_tools

Background tools are simple

Some long-running tools can start in the background so the agent can keep working. There is no detached job queue, polling API, or job-control surface; the current run still owns the task, and the completion is sent back to the model when it finishes.

no_token_streaming

No token streaming

Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, background, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal.

@@ -184,7 +184,7 @@

Use

Features

Filesystem tools

read, write, batched exact-replacement edit, search, list, and glob with root-scoped path policies.

-
JSONL search

Opt-in jsonl_search for structured line-delimited data, with ripgrep prefiltering, field projection, and where filters.

+
JSONL search

Opt-in jsonl_search for structured line-delimited data, with ripgrep prefiltering, field projection, equality/contains/regex/range where filters, and field-level snippets from large multiline string values.

Bash prototype tool

Opt-in BashTool for exploratory shell commands. It is lightweight, custom-registration only, and is not included in the default or built-in tool set.

Provider adapters

Built-in OpenAI, Anthropic, and OpenRouter adapters, plus public model/session protocols for implementing another provider.

Custom typed tools

Define sync or async ToolSpec handlers with Pydantic argument models, normalized ToolResult envelopes, sequential/background/approval flags, and per-tool retry settings.

diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index 6b7c4c3..2879e84 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -55,7 +55,7 @@

Snapshot

runtime Python files under thinharness/, including the tools package.
- 7,892 + 8,197 README-stated framework LOC, intentionally small enough to inspect, adapt, and fork.
@@ -487,7 +487,7 @@

Extras

JSONL search FileTools.__init__ creates self.jsonl, and FileTools.specs() includes self.jsonl.spec(). - Specialized search over large JSONL content stores: ripgrep prefilter, field paths, where filters, projection, row limits, and truncation through FileTools spill behavior. + Specialized search over large JSONL content stores: ripgrep row prefiltering, field paths, equality/contains/regex/range filters, projection, field-level snippets, row limits, and truncation through FileTools spill behavior. Skills @@ -930,7 +930,7 @@

Implementation Deep Dive

JSONL search FileTools owns the JsonlSearch instance and exposes its spec with the filesystem built-ins. - search_support.py holds shared ripgrep parsing, glob validation, containment filtering, and search-root helpers used by both text search and JSONL search. + search_support.py holds shared ripgrep parsing, glob validation, containment filtering, and search-root helpers used by both text search and JSONL search; jsonl.py owns structured field projection, range filters, and field snippet rendering. Subagents diff --git a/docs/site/index.html b/docs/site/index.html index 98d962d..26a7d24 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -36,7 +36,7 @@

A minimal, opinionated agent harness.
install.sh
-
copy$ uv add thinharness
# or: pip install thinharness
# requires python 3.11+

✓ resolved · 22 files · 7,892 LOC
+
copy$ uv add thinharness
# or: pip install thinharness
# requires python 3.11+

✓ resolved · 22 files · 8,197 LOC

@@ -47,7 +47,7 @@

A minimal, opinionated agent harness.
purpose_built

Purpose-built agents

ThinHarness is for bounded agent loops inside software you control, not open-ended interactive assistants.

no_bash

No bash by default

Bash stays out of the default tools, with an opt-in BashTool only for prototyping before typed tools.

skills

Skills are tools, not auto-discovery

Skills live in directories you point at explicitly. The agent calls them like any other tool. No magic scan, no marketplace.

-
search

Search is a top priority

Ripgrep exposed as compact grouped results, tuned for documents and business workflows — plus a JSONL variant for structured data.

+
search

Search is a top priority

Ripgrep exposed as compact grouped results, tuned for documents and business workflows — plus JSONL search with field projection, range filters, and multiline snippets.

parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs reliability beyond a single loop — majority vote, ensembled extraction.

background_tools

Background tools are simple

Long-running tools can start in the background, but the current run still owns the task and receives completion.

diff --git a/docs/table.md b/docs/table.md index aab6c16..2b92884 100644 --- a/docs/table.md +++ b/docs/table.md @@ -13,7 +13,7 @@ the command beside each row. To reproduce locally, clone each upstream repo at the pinned commit and run the command shown below. Measured 2026-06-15 for upstream libraries. ThinHarness was remeasured from the -current working tree on 2026-06-15. +current working tree on 2026-06-19. ## LOC Commands diff --git a/tests/test_file_tools.py b/tests/test_file_tools.py index 2a4056d..b2b9c6f 100644 --- a/tests/test_file_tools.py +++ b/tests/test_file_tools.py @@ -366,6 +366,13 @@ def test_file_tools_validate_glob_selectors(tmp_path: Path) -> None: assert not result.ok assert result.metadata["error_type"] == "PathValidationError" +def test_jsonl_search_schema_exposes_range_and_field_search_arguments(tmp_path: Path) -> None: + jsonl_schema = next(tool.response_tool() for tool in FileTools(tmp_path).specs() if tool.name == "jsonl_search") + jsonl_schema_text = json.dumps(jsonl_schema) + + for token in ["gt", "gte", "lt", "lte", "number", "date", "field_searches", "context_lines", "max_line_chars"]: + assert f'"{token}"' in jsonl_schema_text + def test_search_groups_document_results_by_path_and_line(tmp_path: Path) -> None: (tmp_path / "claims").mkdir() (tmp_path / "policies").mkdir() @@ -532,6 +539,47 @@ def test_jsonl_search_uses_ripgrep_prefilter(tmp_path: Path) -> None: assert "rows_matched: 1" in result.content assert ' 3: {"id": 3}' in result.content +def test_jsonl_search_non_range_where_operators_after_compile_refactor(tmp_path: Path) -> None: + rows = [ + {"id": 1, "status": "open", "owner": "alice", "priority": "high"}, + {"id": 2, "status": "closed", "owner": None, "priority": "low"}, + {"id": 3, "status": "open", "priority": "medium"}, + ] + (tmp_path / "events.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + tools = FileTools(tmp_path) + + ne = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "status", "op": "ne", "value": "closed"}], + "fields": {"id": 0}, + }) + in_filter = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "priority", "op": "in", "values": ["high", "medium"]}], + "fields": {"id": 0}, + }) + exists = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "owner", "op": "exists"}], + "fields": {"id": 0}, + }) + + assert ne.ok, ne.content + assert "rows_matched: 2" in ne.content + assert ' 1: {"id": 1}' in ne.content + assert ' 3: {"id": 3}' in ne.content + assert '"id": 2' not in ne.content + assert in_filter.ok, in_filter.content + assert "rows_matched: 2" in in_filter.content + assert ' 1: {"id": 1}' in in_filter.content + assert ' 3: {"id": 3}' in in_filter.content + assert '"id": 2' not in in_filter.content + assert exists.ok, exists.content + assert "rows_matched: 1" in exists.content + assert ' 1: {"id": 1}' in exists.content + assert '"id": 2' not in exists.content + assert '"id": 3' not in exists.content + def test_jsonl_search_path_accepts_recursive_directory(tmp_path: Path) -> None: nested = tmp_path / "logs" / "nested" nested.mkdir(parents=True) @@ -609,6 +657,491 @@ def partial(*args, **kwargs): assert result.metadata["warning"] == "ripgrep returned 2; showing parsed partial matches" assert "secret" not in json.dumps(result.metadata) +def test_jsonl_search_field_search_returns_matching_internal_lines(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text( + json.dumps({ + "state_index": 11, + "url": "https://example.test", + "accessibility_tree": "\n".join([ + "[a1] button 'Save'", + "[a2] menuitem 'Edit personal filters'", + "[a3] menuitem '-- None --'", + "[a4] menuitem 'Incident Portal'", + ]), + }) + + "\n", + encoding="utf-8", + ) + + result = FileTools(tmp_path).jsonl_search({ + "path": "states.jsonl", + "where": [{"field": "state_index", "op": "eq", "value": "11"}], + "fields": {"state_index": 0, "url": 0}, + "field_searches": [{"field": "accessibility_tree", "query": "Incident|-- None --|Edit personal filters", "regex": True}], + }) + + assert result.ok, result.content + assert "field_searches: accessibility_tree" in result.content + assert ' 1: {"state_index": 11, "url": "https://example.test"}' in result.content + assert " accessibility_tree matches:" in result.content + assert " 2: [a2] menuitem 'Edit personal filters'" in result.content + assert " 3: [a3] menuitem '-- None --'" in result.content + assert " 4: [a4] menuitem 'Incident Portal'" in result.content + assert "Save" not in result.content + +def test_jsonl_search_field_search_without_fields_does_not_render_whole_row(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text( + json.dumps({"id": 1, "blob": "alpha\nneedle\nomega", "secret": "do not print"}) + "\n", + encoding="utf-8", + ) + + result = FileTools(tmp_path).jsonl_search({"path": "*.jsonl", "field_searches": [{"field": "blob", "query": "needle"}]}) + + assert result.ok, result.content + assert ' 1: {}' in result.content + assert " 2: needle" in result.content + assert "secret" not in result.content + assert "alpha" not in result.content + +def test_jsonl_search_field_search_top_level_query_remains_row_prefilter(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text( + "\n".join([ + json.dumps({"id": 1, "kind": "candidate", "blob": "target internal line"}), + json.dumps({"id": 2, "kind": "other", "blob": "target internal line"}), + ]) + + "\n", + encoding="utf-8", + ) + + result = FileTools(tmp_path).jsonl_search({ + "query": "candidate", + "path": "*.jsonl", + "fields": {"id": 0}, + "field_searches": [{"field": "blob", "query": "target"}], + }) + + assert result.ok, result.content + assert "rows_matched: 1" in result.content + assert ' 1: {"id": 1}' in result.content + assert '"id": 2' not in result.content + +def test_jsonl_search_field_search_miss_with_fields_keeps_output_compact(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "blob": "alpha"}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "fields": {"id": 0}, + "field_searches": [{"field": "blob", "query": "needle"}], + }) + + assert result.ok, result.content + assert ' 1: {"id": 1}' in result.content + assert "blob matches" not in result.content + +def test_jsonl_search_field_search_multiple_and_duplicate_searches_render_in_order(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text( + json.dumps({"id": 1, "body": "alpha\nbeta\nALPHA", "thought": "first\nfilters\nlast"}) + "\n", + encoding="utf-8", + ) + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "fields": {"id": 0}, + "field_searches": [ + {"field": "body", "query": "alpha"}, + {"field": "thought", "query": "filters"}, + {"field": "body", "query": "beta"}, + ], + }) + + assert result.ok, result.content + first = result.content.index(" body matches #1 (query='alpha'):") + second = result.content.index(" thought matches:") + third = result.content.index(" body matches #2 (query='beta'):") + assert first < second < third + assert " 1: alpha" in result.content + assert " 3: ALPHA" in result.content + assert " 2: beta" in result.content + +def test_jsonl_search_field_search_duplicate_same_query_labels_are_distinguishable(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "body": "alpha\nmiddle\nalpha"}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "fields": {"id": 0}, + "field_searches": [ + {"field": "body", "query": "alpha", "context_lines": 0}, + {"field": "body", "query": "alpha", "context_lines": 1}, + ], + }) + + assert result.ok, result.content + assert " body matches #1 (query='alpha'):" in result.content + assert " body matches #2 (query='alpha'):" in result.content + +def test_jsonl_search_field_search_context_merges_and_limits_matches(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text( + json.dumps({"id": 1, "blob": "\n".join(["before", "target one", "middle", "target two", "after", "target three"])}) + "\n", + encoding="utf-8", + ) + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [{"field": "blob", "query": "target", "context_lines": 1, "max_matches": 2}], + }) + + assert result.ok, result.content + assert " 1: before" in result.content + assert " 2: target one" in result.content + assert " 3: middle" in result.content + assert " 4: target two" in result.content + assert " 5: after" in result.content + assert "target three" not in result.content + assert " ... 1 more match(es)" in result.content + assert result.content.count(" 3: middle") == 1 + +def test_jsonl_search_field_search_does_not_count_context_visible_match_as_omitted(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "blob": "target one\ntarget two\nlast"}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [{"field": "blob", "query": "target", "context_lines": 1, "max_matches": 1}], + }) + + assert result.ok, result.content + assert " 1: target one" in result.content + assert " 2: target two" in result.content + assert "more match(es)" not in result.content + +def test_jsonl_search_field_search_truncates_and_respects_case_sensitive(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "blob": "Needle abcdef\nneedle ghijkl"}) + "\n", encoding="utf-8") + + insensitive = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [{"field": "blob", "query": "needle", "max_line_chars": 10}], + }) + sensitive = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [{"field": "blob", "query": "needle", "case_sensitive": True}], + }) + + assert insensitive.ok, insensitive.content + assert " 1: Needle abc…" in insensitive.content + assert " 2: needle ghi…" in insensitive.content + assert sensitive.ok, sensitive.content + assert "Needle" not in sensitive.content + assert " 2: needle ghijkl" in sensitive.content + +def test_jsonl_search_field_search_string_miss_without_fields_renders_plain_none(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "blob": "alpha"}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({"path": "*.jsonl", "field_searches": [{"field": "blob", "query": "needle"}]}) + + assert result.ok, result.content + assert " 1: {}\n blob matches: none" in result.content + +def test_jsonl_search_field_search_regex_is_case_insensitive_by_default(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "blob": "Incident Portal"}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [{"field": "blob", "query": "incident", "regex": True}], + }) + + assert result.ok, result.content + assert " 1: Incident Portal" in result.content + +def test_jsonl_search_field_search_regex_respects_case_sensitive(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "blob": "Needle\nneedle"}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [{"field": "blob", "query": "needle", "regex": True, "case_sensitive": True}], + }) + + assert result.ok, result.content + assert "Needle" not in result.content + assert " 2: needle" in result.content + +def test_jsonl_search_field_search_invalid_regex_and_field_path_fail_clearly(tmp_path: Path) -> None: + tools = FileTools(tmp_path) + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "blob": "needle"}) + "\n", encoding="utf-8") + + invalid_regex = tools.jsonl_search({"path": "*.jsonl", "field_searches": [{"field": "blob", "query": "[", "regex": True}]}) + invalid_path = tools.jsonl_search({"path": "*.jsonl", "field_searches": [{"field": "blob[", "query": "needle"}]}) + + assert not invalid_regex.ok + assert invalid_regex.content.startswith("invalid field_search regex:") + assert not invalid_path.ok + assert invalid_path.content.startswith("invalid field path:") + +def test_jsonl_search_field_search_missing_and_non_string_fields_are_compact(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text( + "\n".join([ + json.dumps({"id": 1, "blob": 123}), + json.dumps({"id": 2, "blob": None}), + json.dumps({"id": 3}), + ]) + + "\n", + encoding="utf-8", + ) + + result = FileTools(tmp_path).jsonl_search({"path": "*.jsonl", "field_searches": [{"field": "blob", "query": "needle"}]}) + + assert result.ok, result.content + assert "rows_matched: 3" in result.content + assert " 1: {}\n blob matches: none (non-string field)" in result.content + assert " 2: {}\n blob matches: none (null field)" in result.content + assert " 3: {}\n blob matches: none (missing field)" in result.content + +def test_jsonl_search_field_search_suppresses_miss_notes_when_sibling_matches(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "body": "needle", "other": 123}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [ + {"field": "body", "query": "needle"}, + {"field": "other", "query": "needle"}, + ], + }) + + assert result.ok, result.content + assert " body matches:" in result.content + assert "other matches" not in result.content + +def test_jsonl_search_field_search_single_line_field_with_context(tmp_path: Path) -> None: + (tmp_path / "states.jsonl").write_text(json.dumps({"id": 1, "body": "needle"}) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "field_searches": [{"field": "body", "query": "needle", "context_lines": 3}], + }) + + assert result.ok, result.content + assert " 1: needle" in result.content + assert " 0:" not in result.content + +def test_jsonl_search_field_search_combines_with_range_where_filters(tmp_path: Path) -> None: + rows = [ + {"id": 1, "score": 2, "blob": "target line"}, + {"id": 2, "score": "bad", "blob": "target line"}, + {"id": 3, "score": 0, "blob": "target line"}, + ] + (tmp_path / "states.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "score", "op": "gt", "value": "1", "type": "number"}], + "fields": {"id": 0}, + "field_searches": [{"field": "blob", "query": "target"}], + }) + + assert result.ok, result.content + assert "rows_matched: 1" in result.content + assert "compare_warnings: 1 row(s) had non-comparable values" in result.content + assert result.metadata["compare_warnings"] == 1 + assert ' 1: {"id": 1}' in result.content + assert " 1: target line" in result.content + +def test_jsonl_search_number_range_filters_match_json_numbers_only(tmp_path: Path) -> None: + rows = [ + {"id": 1, "score": 9.5}, + {"id": 2, "score": 8}, + {"id": 3, "score": "9.5"}, + {"id": 4, "score": True}, + {"id": 5, "score": float("nan")}, + {"id": 6, "score": -3}, + {"id": 7}, + {"id": 8, "score": None}, + {"id": 9, "score": {"nested": 9}}, + {"id": 10, "score": [9]}, + ] + (tmp_path / "events.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "score", "op": "gt", "value": "8", "type": "number"}], + "fields": {"id": 0}, + }) + + assert result.ok, result.content + assert "scope: path=*.jsonl" in result.content + assert "where: score gt '8' (number)" in result.content + assert "rows_matched: 1" in result.content + assert ' 1: {"id": 1}' in result.content + assert '"id": 2' not in result.content + assert result.metadata["compare_warnings"] == 7 + assert "compare_warnings: 7 row(s) had non-comparable values" in result.content + +def test_jsonl_search_number_range_strictness_and_negative_targets(tmp_path: Path) -> None: + (tmp_path / "events.jsonl").write_text( + "\n".join(json.dumps({"id": index, "score": score}) for index, score in enumerate([-2, -1, 0], start=1)) + "\n", + encoding="utf-8", + ) + tools = FileTools(tmp_path) + + gt = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "score", "op": "gt", "value": "-1", "type": "number"}], + "fields": {"id": 0}, + }) + lt = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "score", "op": "lt", "value": "-1", "type": "number"}], + "fields": {"id": 0}, + }) + + assert gt.ok + assert "rows_matched: 1" in gt.content + assert ' 3: {"id": 3}' in gt.content + assert lt.ok + assert "rows_matched: 1" in lt.content + assert ' 1: {"id": 1}' in lt.content + assert "compare_warnings" not in gt.content + assert "compare_warnings" not in gt.metadata + +def test_jsonl_search_number_range_filters_compare_large_integers_exactly(tmp_path: Path) -> None: + target = 9_007_199_254_740_993 + rows = [ + {"id": 1, "counter": target - 1}, + {"id": 2, "counter": target}, + {"id": 3, "counter": target + 1}, + ] + (tmp_path / "events.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + tools = FileTools(tmp_path) + + cases = [ + ("gt", [3]), + ("gte", [2, 3]), + ("lt", [1]), + ("lte", [1, 2]), + ] + for op, expected_ids in cases: + result = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "counter", "op": op, "value": str(target), "type": "number"}], + "fields": {"id": 0}, + }) + + assert result.ok, result.content + assert f"rows_matched: {len(expected_ids)}" in result.content + for expected_id in expected_ids: + assert f'{{"id": {expected_id}}}' in result.content + for unexpected_id in {1, 2, 3} - set(expected_ids): + assert f'{{"id": {unexpected_id}}}' not in result.content + +def test_jsonl_search_rejects_invalid_range_filter_definitions_before_scanning(tmp_path: Path) -> None: + tools = FileTools(tmp_path) + bad_filters = [ + [{"field": "score", "op": "gt", "value": "NaN", "type": "number"}], + [{"field": "score", "op": "gt", "value": "Infinity", "type": "number"}], + [{"field": "score", "op": "gt", "value": "-Infinity", "type": "number"}], + [{"field": "score", "op": "gt", "value": "", "type": "number"}], + [{"field": "score", "op": "gt", "value": "1"}], + [{"field": "score", "op": "gt", "value": "1", "type": "number", "values": ["1"]}], + [{"field": "score", "op": "eq", "value": "1", "type": "number"}], + [{"field": "published_at", "op": "lt", "value": "not-a-date", "type": "date"}], + ] + + for where in bad_filters: + result = tools.jsonl_search({"path": "*.jsonl", "where": where}) + assert not result.ok + assert result.content.startswith("invalid where filter:"), where + +def test_jsonl_search_date_range_filters_use_declared_date_semantics(tmp_path: Path) -> None: + rows = [ + {"id": 1, "published_at": "2026-06-11"}, + {"id": 2, "published_at": "2026-06-12"}, + {"id": 3, "published_at": "2026-06-12T15:30:00Z"}, + {"id": 4, "published_at": "2026-06-12T23:30:00-04:00"}, + {"id": 5, "published_at": "2026-06-13"}, + ] + (tmp_path / "events.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + tools = FileTools(tmp_path) + + lte = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "published_at", "op": "lte", "value": "2026-06-12", "type": "date"}], + "fields": {"id": 0}, + }) + strict_gt = tools.jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "published_at", "op": "gt", "value": "2026-06-12", "type": "date"}], + "fields": {"id": 0}, + }) + + assert lte.ok, lte.content + assert "where: published_at lte '2026-06-12' (date)" in lte.content + assert "rows_matched: 4" in lte.content + assert ' 3: {"id": 3}' in lte.content + assert ' 4: {"id": 4}' in lte.content + assert strict_gt.ok + assert "rows_matched: 1" in strict_gt.content + assert ' 5: {"id": 5}' in strict_gt.content + +def test_jsonl_search_datetime_range_filters_compare_like_awareness(tmp_path: Path) -> None: + rows = [ + {"id": 1, "published_at": "2026-06-12T08:00:00"}, + {"id": 2, "published_at": "2026-06-12T10:00:00"}, + {"id": 3, "published_at": "2026-06-12T15:00:00Z"}, + {"id": 4, "published_at": "not-a-date"}, + ] + (tmp_path / "events.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "where": [{"field": "published_at", "op": "gte", "value": "2026-06-12T09:00:00", "type": "date"}], + "fields": {"id": 0}, + }) + + assert result.ok, result.content + assert "rows_matched: 1" in result.content + assert ' 2: {"id": 2}' in result.content + assert result.metadata["compare_warnings"] == 2 + +def test_jsonl_search_range_warning_counts_candidate_rows_once(tmp_path: Path) -> None: + rows = [ + {"id": 1, "kind": "keep", "score": "bad", "other": "bad"}, + {"id": 2, "kind": "skip", "score": "bad", "other": "bad"}, + {"id": 3, "kind": "keep", "score": 2, "other": "bad"}, + ] + (tmp_path / "events.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "path": "*.jsonl", + "where": [ + {"field": "kind", "op": "eq", "value": "keep"}, + {"field": "score", "op": "gte", "value": "1", "type": "number"}, + {"field": "other", "op": "gte", "value": "1", "type": "number"}, + ], + "fields": {"id": 0}, + }) + + assert result.ok, result.content + assert "rows_matched: 0" in result.content + assert result.metadata["compare_warnings"] == 2 + +def test_jsonl_search_compare_warnings_coexist_with_ripgrep_warnings(tmp_path: Path, monkeypatch) -> None: + def partial(*args, **kwargs): + stdout = "\n".join([ + _rg_match("events.jsonl", 1, '{"id":1,"msg":"login","score":"bad"}'), + "rg: ./restricted: Permission denied", + ]) + return subprocess.CompletedProcess(args[0], 2, stdout=stdout) + + monkeypatch.setattr("subprocess.run", partial) + (tmp_path / "events.jsonl").write_text('{"id":1,"msg":"login","score":"bad"}\n', encoding="utf-8") + + result = FileTools(tmp_path).jsonl_search({ + "query": "login", + "path": "*.jsonl", + "where": [{"field": "score", "op": "gte", "value": "1", "type": "number"}], + }) + + assert result.ok, result.content + assert result.metadata["warning"] == "ripgrep returned 2; showing parsed partial matches" + assert result.metadata["compare_warnings"] == 1 + def test_spill_output_uses_thinharness_directory_and_read_guidance(tmp_path: Path) -> None: (tmp_path / "notes.txt").write_text("\n".join(f"hit {i}" for i in range(100)), encoding="utf-8") tools = FileTools(tmp_path, max_tool_chars=120) diff --git a/thinharness/defaults.py b/thinharness/defaults.py index 88625ac..b3e150f 100644 --- a/thinharness/defaults.py +++ b/thinharness/defaults.py @@ -62,7 +62,9 @@ - Use jsonl_search for saved JSONL files when you need structured rows or selected fields. - Pass path to scope to a JSONL file, a directory of JSONL files, or a glob; omit path to search all readable JSONL files. - Pass query for an optional ripgrep prefilter before field and where filtering. -- Use fields to return only the keys needed for the next decision.""" +- Use fields to return only the keys needed for the next decision. +- Use where range filters with op gt/gte/lt/lte and type number/date for explicit scalar comparisons. +- Use field_searches to extract matching lines from large multiline string fields after rows are selected.""" DEFAULT_PARALLEL_LLM_DESCRIPTION_BASE = ( "Run N independent prompts as one-shot LLM completions in parallel. Each call is stateless: no tools, no memory, no continuation " diff --git a/thinharness/tools/__init__.py b/thinharness/tools/__init__.py index d5207b6..42ef3db 100644 --- a/thinharness/tools/__init__.py +++ b/thinharness/tools/__init__.py @@ -16,7 +16,7 @@ ) from .bash import BashArgs, BashTool from .filesystem import FileTools, builtin_tools -from .jsonl import JsonlSearch, JsonlSearchArgs, JsonlWhereFilter +from .jsonl import JsonlFieldSearch, JsonlSearch, JsonlSearchArgs, JsonlWhereFilter from .mcp import MCPDependencyError, MCPError, MCPServer, MCPServerSSE, MCPServerStdio, MCPServerStreamableHTTP from .parallel_llm import FilePromptSource, InlinePromptSource, ParallelLlmArgs, ParallelLlmTool, create_parallel_llm_tool from .skills import Skill, SkillRegistry @@ -28,6 +28,7 @@ "Json", "JsonlSearch", "JsonlSearchArgs", + "JsonlFieldSearch", "JsonlWhereFilter", "MCPDependencyError", "MCPError", diff --git a/thinharness/tools/jsonl.py b/thinharness/tools/jsonl.py index 98b497f..40c3fce 100644 --- a/thinharness/tools/jsonl.py +++ b/thinharness/tools/jsonl.py @@ -3,10 +3,13 @@ from __future__ import annotations import json +import math import re import subprocess from collections.abc import Callable, Iterator from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Annotated, Any, Literal @@ -37,9 +40,22 @@ class JsonlWhereFilter(StrictArgs): """One JSONL where filter.""" field: str - op: Literal["eq", "ne", "in", "contains", "regex", "exists"] + op: Literal["eq", "ne", "in", "contains", "regex", "exists", "gt", "gte", "lt", "lte"] value: str | None = None values: list[str] | None = None + type: Literal["number", "date"] | None = None + + +class JsonlFieldSearch(StrictArgs): + """Search inside one string field on selected JSONL rows.""" + + field: str = Field(min_length=1) + query: str = Field(min_length=1) + regex: bool = False + case_sensitive: bool = False + context_lines: int = Field(default=0, ge=0) + max_matches: int = Field(default=20, ge=1) + max_line_chars: int = Field(default=300, ge=1) class JsonlSearchArgs(StrictArgs): @@ -52,6 +68,10 @@ class JsonlSearchArgs(StrictArgs): description="Map of jq-style field path to max chars (0 = no truncation). If omitted, return the whole row.", ) where: list[JsonlWhereFilter] = Field(default_factory=list, description="Filters AND-ed together.") + field_searches: list[JsonlFieldSearch] = Field( + default_factory=list, + description="Search inside selected string fields and return matching internal field lines/snippets.", + ) max_files: int = Field(default=100, ge=1) max_matches_per_file: int = Field(default=25, ge=1) timeout: int | None = Field(default=None, ge=1) @@ -67,6 +87,64 @@ class _CandidateScan: error: ToolResult | None = None +_RANGE_OPS = {"gt", "gte", "lt", "lte"} + + +@dataclass(frozen=True) +class _DateValue: + """Parsed date-like value plus whether the source was date-only.""" + + value: date | datetime + date_only: bool + + +@dataclass(frozen=True) +class _CompiledWhere: + """Pre-validated JSONL where filter.""" + + field: str + segments: list[str | int] + op: str + value: str | None + values: list[str] | None + compare_type: Literal["number", "date"] | None + target: Any = None + + +@dataclass(frozen=True) +class _WhereResult: + """One row's where result, including one-shot comparison warning state.""" + + passed: bool + compare_warning: bool = False + + +@dataclass(frozen=True) +class _CompiledFieldSearch: + """Pre-validated field-level string search.""" + + field: str + occurrence: int + segments: list[str | int] + query: str + pattern: re.Pattern[str] | None + case_sensitive: bool + context_lines: int + max_matches: int + max_line_chars: int + show_query_label: bool + + +@dataclass(frozen=True) +class _RenderedFieldSearch: + """Rendered field search lines for one selected row.""" + + label: str + lines: list[str] + omitted_matches: int = 0 + note: str | None = None + + class JsonlSearch: """Search JSONL files with optional ripgrep prefiltering and field projection.""" @@ -101,7 +179,17 @@ def search(self, args: JsonlSearchArgs | Json) -> ToolResult: query = args.query path = args.path fields = args.fields - where = [item.model_dump(exclude_none=True) for item in args.where] + where_for_display = [item.model_dump(exclude_none=True) for item in args.where] + try: + where = _compile_where(args.where) + except ValueError as exc: + return ToolResult(False, f"invalid where filter: {exc}") + try: + field_searches = _compile_field_searches(args.field_searches) + except re.error as exc: + return ToolResult(False, f"invalid field_search regex: {exc}") + except ValueError as exc: + return ToolResult(False, f"invalid field path: {exc}") max_files = args.max_files max_matches_per_file = args.max_matches_per_file timeout = args.timeout or self.rg_timeout @@ -114,9 +202,10 @@ def search(self, args: JsonlSearchArgs | Json) -> ToolResult: shown: dict[str, list[tuple[int, Any]]] = {} row_counts: dict[str, int] = {} json_errors = 0 + compare_warnings = 0 total_files = 0 total_rows = 0 - for path, line_number, line_text in scan.candidates: + for file_path, line_number, line_text in scan.candidates: if not line_text.strip(): continue try: @@ -124,20 +213,20 @@ def search(self, args: JsonlSearchArgs | Json) -> ToolResult: except json.JSONDecodeError: json_errors += 1 continue - try: - if not _where_passes(row, where): - continue - except ValueError as exc: - return ToolResult(False, f"invalid where filter: {exc}") - if path not in row_counts: + where_result = _where_passes(row, where) + if where_result.compare_warning: + compare_warnings += 1 + if not where_result.passed: + continue + if file_path not in row_counts: total_files += 1 - row_counts[path] = 0 - row_counts[path] += 1 + row_counts[file_path] = 0 + row_counts[file_path] += 1 total_rows += 1 - if path in shown or len(shown) < max_files: - shown.setdefault(path, []) - if len(shown[path]) < max_matches_per_file: - shown[path].append((line_number, row)) + if file_path in shown or len(shown) < max_files: + shown.setdefault(file_path, []) + if len(shown[file_path]) < max_matches_per_file: + shown[file_path].append((line_number, row)) shown_files = sorted(shown.items()) @@ -146,25 +235,38 @@ def search(self, args: JsonlSearchArgs | Json) -> ToolResult: f" query: {query or '(none)'}", f" scope: path={path}", ] - if where: - header.append(f" where: {_describe_where(where)}") + if where_for_display: + header.append(f" where: {_describe_where(where_for_display)}") if fields: header.append(f" fields: {', '.join(fields)}") + if field_searches: + header.append(f" field_searches: {', '.join(search.field for search in field_searches)}") header.append(f" files: {total_files} total, {len(shown_files)} shown") header.append(f" rows_matched: {total_rows}") + if compare_warnings: + header.append(f" compare_warnings: {compare_warnings} row(s) had non-comparable values") if json_errors: header.append(f" json_parse_errors: {json_errors}") body = [""] - for path, rows in shown_files: + for file_path, rows in shown_files: rows.sort(key=lambda lr: lr[0]) - body.append(path) + body.append(file_path) for line_number, row in rows: try: - projected = _project_fields(row, fields) if fields else row + projected = _project_fields(row, fields) if fields else ({} if field_searches else row) except ValueError as exc: return ToolResult(False, f"invalid field path: {exc}") body.append(f" {line_number}: {json.dumps(projected, ensure_ascii=False, default=str)}") - omitted_rows = row_counts[path] - len(rows) + if field_searches: + for rendered in _field_search_matches(row, field_searches, show_empty=not fields): + if rendered.lines: + body.append(f" {rendered.label}:") + body.extend(f" {line}" for line in rendered.lines) + if rendered.omitted_matches: + body.append(f" ... {rendered.omitted_matches} more match(es)") + elif rendered.note: + body.append(f" {rendered.label}: {rendered.note}") + omitted_rows = row_counts[file_path] - len(rows) if omitted_rows: body.append(f" ... {omitted_rows} more row(s)") if total_files > max_files: @@ -172,6 +274,8 @@ def search(self, args: JsonlSearchArgs | Json) -> ToolResult: result = self._truncate("\n".join(header + body), prefix="jsonl_search", max_chars=limit_chars) result.metadata.update(scan.metadata) + if compare_warnings: + result.metadata["compare_warnings"] = compare_warnings return result def _candidates(self, query: str, path: str, timeout: int) -> _CandidateScan: @@ -298,6 +402,7 @@ def _display_path(root: Path, path: Path) -> str: _MISSING = object() +_DATE_ONLY_RE = re.compile(r"\d{4}-\d{2}-\d{2}") _PATH_TOKEN = re.compile( r'\.?(?:([A-Za-z_][A-Za-z0-9_]*)|\[(-?\d+)\]|\["([^"]*)"\]|\[\'([^\']*)\'\])' ) @@ -344,6 +449,158 @@ def _get_field_by_path(obj: Any, segments: list[str | int]) -> Any: return cur +def _compile_where(where: list[JsonlWhereFilter]) -> list[_CompiledWhere]: + """Validate where filters once and parse range targets before scanning.""" + compiled = [] + for filt in where: + field = filt.field + op = filt.op + if not field or not op: + raise ValueError(f"where filter missing field or op: {filt.model_dump(exclude_none=True)}") + segments = _parse_jq_path(field) + if op in _RANGE_OPS: + if filt.value is None: + raise ValueError(f"op {op!r} requires 'value'") + if filt.value == "": + raise ValueError(f"op {op!r} requires non-empty 'value'") + if filt.type is None: + raise ValueError(f"op {op!r} requires 'type'") + if filt.values is not None: + raise ValueError(f"op {op!r} does not accept 'values'") + compiled.append( + _CompiledWhere( + field=field, + segments=segments, + op=op, + value=filt.value, + values=None, + compare_type=filt.type, + target=_parse_range_target(filt.type, filt.value), + ) + ) + continue + if filt.type is not None: + raise ValueError(f"op {op!r} does not accept 'type'") + if op in {"eq", "ne", "contains", "regex"} and filt.value is None: + raise ValueError(f"op {op!r} requires 'value'") + if op == "in" and filt.values is None: + raise ValueError("op 'in' requires 'values'") + compiled.append( + _CompiledWhere( + field=field, + segments=segments, + op=op, + value=filt.value, + values=filt.values, + compare_type=None, + ) + ) + return compiled + + +def _compile_field_searches(field_searches: list[JsonlFieldSearch]) -> list[_CompiledFieldSearch]: + """Validate field searches once before scanning rows.""" + field_counts: dict[str, int] = {} + for search in field_searches: + field_counts[search.field] = field_counts.get(search.field, 0) + 1 + + compiled = [] + field_occurrences: dict[str, int] = {} + for search in field_searches: + occurrence = field_occurrences.get(search.field, 0) + 1 + field_occurrences[search.field] = occurrence + flags = 0 if search.case_sensitive else re.IGNORECASE + compiled.append( + _CompiledFieldSearch( + field=search.field, + occurrence=occurrence, + segments=_parse_jq_path(search.field), + query=search.query, + pattern=re.compile(search.query, flags) if search.regex else None, + case_sensitive=search.case_sensitive, + context_lines=search.context_lines, + max_matches=search.max_matches, + max_line_chars=search.max_line_chars, + show_query_label=field_counts[search.field] > 1, + ) + ) + return compiled + + +def _field_search_matches(row: Any, searches: list[_CompiledFieldSearch], *, show_empty: bool) -> list[_RenderedFieldSearch]: + """Return rendered snippet blocks for every matching field search on one row.""" + rendered = [] + for search in searches: + value = _get_field_by_path(row, search.segments) + label = _field_search_label(search) + if value is _MISSING: + if show_empty: + rendered.append(_RenderedFieldSearch(label, [], note="none (missing field)")) + continue + if value is None: + if show_empty: + rendered.append(_RenderedFieldSearch(label, [], note="none (null field)")) + continue + if not isinstance(value, str): + if show_empty: + rendered.append(_RenderedFieldSearch(label, [], note="none (non-string field)")) + continue + lines = value.splitlines() + matching_indexes = [index for index, line in enumerate(lines) if _field_line_matches(line, search)] + if not matching_indexes: + if show_empty: + rendered.append(_RenderedFieldSearch(label, [], note="none")) + continue + + displayed_matches = matching_indexes[: search.max_matches] + ranges = _line_ranges_for_matches(displayed_matches, total_lines=len(lines), context_lines=search.context_lines) + displayed_indexes = {index for start, end in ranges for index in range(start, end)} + snippet_lines = [ + f"{index + 1}: {_truncate_internal_line(lines[index], search.max_line_chars)}" + for start, end in ranges + for index in range(start, end) + ] + omitted_matches = sum(1 for index in matching_indexes[search.max_matches :] if index not in displayed_indexes) + rendered.append(_RenderedFieldSearch(label, snippet_lines, omitted_matches=omitted_matches)) + if any(item.lines for item in rendered): + return [item for item in rendered if item.lines] + return rendered + + +def _field_search_label(search: _CompiledFieldSearch) -> str: + """Return a stable output label for a field search.""" + if search.show_query_label: + return f"{search.field} matches #{search.occurrence} (query={search.query!r})" + return f"{search.field} matches" + + +def _field_line_matches(line: str, search: _CompiledFieldSearch) -> bool: + """Return whether one internal field line matches a compiled field search.""" + if search.pattern is not None: + return search.pattern.search(line) is not None + if search.case_sensitive: + return search.query in line + return search.query.casefold() in line.casefold() + + +def _line_ranges_for_matches(matches: list[int], *, total_lines: int, context_lines: int) -> list[tuple[int, int]]: + """Build merged half-open line ranges around primary match indexes.""" + ranges: list[tuple[int, int]] = [] + for index in matches: + start = max(0, index - context_lines) + end = min(total_lines, index + context_lines + 1) + if ranges and start <= ranges[-1][1]: + ranges[-1] = (ranges[-1][0], max(ranges[-1][1], end)) + else: + ranges.append((start, end)) + return ranges + + +def _truncate_internal_line(line: str, max_chars: int) -> str: + """Truncate one rendered field line.""" + return line if len(line) <= max_chars else line[:max_chars] + "…" + + def _apply_where_op(value: Any, op: str, target: Any, targets: Any) -> bool: """Evaluate one where operator against a resolved field value.""" if op == "exists": @@ -364,21 +621,117 @@ def _apply_where_op(value: Any, op: str, target: Any, targets: Any) -> bool: raise ValueError(f"unknown op: {op!r}") -def _where_passes(row: Any, where: list[Json]) -> bool: - """Return True if a row passes every where filter (AND).""" +def _where_passes(row: Any, where: list[_CompiledWhere]) -> _WhereResult: + """Return whether a row passes every where filter (AND).""" for filt in where: - field = filt.get("field") - op = filt.get("op") - if not field or not op: - raise ValueError(f"where filter missing field or op: {filt}") - if op in {"eq", "ne", "contains", "regex"} and "value" not in filt: - raise ValueError(f"op {op!r} requires 'value'") - if op == "in" and "values" not in filt: - raise ValueError("op 'in' requires 'values'") - value = _get_field_by_path(row, _parse_jq_path(field)) - if not _apply_where_op(value, op, filt.get("value"), filt.get("values")): - return False - return True + value = _get_field_by_path(row, filt.segments) + if filt.op in _RANGE_OPS: + if filt.compare_type is None: + raise ValueError(f"range op {filt.op!r} missing compare type") + passed, non_comparable = _apply_range_op(value, filt.op, filt.compare_type, filt.target) + if non_comparable: + return _WhereResult(False, compare_warning=True) + if not passed: + return _WhereResult(False) + continue + if not _apply_where_op(value, filt.op, filt.value, filt.values): + return _WhereResult(False) + return _WhereResult(True) + + +def _parse_range_target(compare_type: Literal["number", "date"], value: str) -> Decimal | _DateValue: + """Parse a range filter target for its declared type.""" + if compare_type == "number": + try: + parsed = Decimal(value) + except InvalidOperation as exc: + raise ValueError(f"number range value must be a finite number: {value!r}") from exc + if parsed.is_nan() or parsed.is_infinite(): + raise ValueError(f"number range value must be a finite number: {value!r}") + return parsed + parsed_date = _parse_date_value(value) + if parsed_date is None: + raise ValueError(f"date range value must be ISO-like: {value!r}") + return parsed_date + + +def _parse_date_value(value: Any) -> _DateValue | None: + """Parse supported ISO-ish date strings.""" + if not isinstance(value, str): + return None + if _DATE_ONLY_RE.fullmatch(value): + try: + return _DateValue(date.fromisoformat(value), date_only=True) + except ValueError: + return None + try: + return _DateValue(datetime.fromisoformat(value.removesuffix("Z") + ("+00:00" if value.endswith("Z") else "")), date_only=False) + except ValueError: + return None + + +def _apply_range_op(value: Any, op: str, compare_type: Literal["number", "date"], target: Any) -> tuple[bool, bool]: + """Evaluate one range operator, returning (passed, non_comparable).""" + if compare_type == "number": + comparable = _number_value(value) + if comparable is None: + return False, True + return _compare_range(comparable, op, target), False + if compare_type == "date": + comparable = _parse_date_value(value) + if comparable is None: + return False, True + compared = _compare_date_range(comparable, op, target) + if compared is None: + return False, True + return compared, False + raise ValueError(f"unknown range type: {compare_type!r}") + + +def _number_value(value: Any) -> Decimal | None: + """Return a comparable JSON number, excluding bool and non-finite float values.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if isinstance(value, float) and not math.isfinite(value): + return None + return Decimal(value) if isinstance(value, int) else Decimal(str(value)) + + +def _compare_date_range(left: _DateValue, op: str, right: _DateValue) -> bool | None: + """Compare parsed date values, returning None for aware/naive datetime mismatch.""" + if left.date_only or right.date_only: + return _compare_range(_calendar_date(left.value), op, _calendar_date(right.value)) + + if not isinstance(left.value, datetime) or not isinstance(right.value, datetime): + return None + if _datetime_is_aware(left.value) != _datetime_is_aware(right.value): + return None + return _compare_range(left.value, op, right.value) + + +def _datetime_is_aware(value: datetime) -> bool: + """Return whether a datetime has an effective timezone offset.""" + return value.utcoffset() is not None + + +def _calendar_date(value: date | datetime) -> date: + """Return the written calendar date for a date or datetime value.""" + if isinstance(value, datetime): + return value.date() + return value + + +def _compare_range(left: Any, op: str, right: Any) -> bool: + """Apply a range operator to already-comparable values.""" + if op == "gt": + return left > right + if op == "gte": + return left >= right + if op == "lt": + return left < right + if op == "lte": + return left <= right + raise ValueError(f"unknown op: {op!r}") def _project_fields(row: Any, fields: dict[str, Any]) -> Json: @@ -407,6 +760,8 @@ def _describe_where(where: list[Json]) -> str: parts.append(f"{filt.get('field')} in {filt.get('values')}") elif op == "exists": parts.append(f"{filt.get('field')} exists") + elif op in _RANGE_OPS: + parts.append(f"{filt.get('field')} {op} {filt.get('value')!r} ({filt.get('type')})") else: parts.append(f"{filt.get('field')} {op} {filt.get('value')!r}") return "; ".join(parts) From adc343a07b42cc421f2cdf1d58d8dd7b3a4a11ba Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Fri, 19 Jun 2026 17:01:28 -0400 Subject: [PATCH 2/2] docs: number implemented JSONL plans --- ...l-search-range-filters.md => 27-jsonl-search-range-filters.md} | 0 ...field-search-snippets.md => 28-jsonl-field-search-snippets.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .plans/{jsonl-search-range-filters.md => 27-jsonl-search-range-filters.md} (100%) rename .plans/{jsonl-field-search-snippets.md => 28-jsonl-field-search-snippets.md} (100%) diff --git a/.plans/jsonl-search-range-filters.md b/.plans/27-jsonl-search-range-filters.md similarity index 100% rename from .plans/jsonl-search-range-filters.md rename to .plans/27-jsonl-search-range-filters.md diff --git a/.plans/jsonl-field-search-snippets.md b/.plans/28-jsonl-field-search-snippets.md similarity index 100% rename from .plans/jsonl-field-search-snippets.md rename to .plans/28-jsonl-field-search-snippets.md