From 5fac68c3f30a0ca3af32f0184dd69e8c6ec7c310 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Fri, 24 Jul 2026 17:30:55 +0200 Subject: [PATCH] Score compatible navigation destinations --- benchmarks/README.md | 8 + benchmarks/cases/go-baseline.yaml | 3 + benchmarks/cases/go-lsp-parity.yaml | 1 + benchmarks/cases/go-precision.yaml | 1 + benchmarks/cases/javascript-baseline.yaml | 1 + benchmarks/cases/php-baseline.yaml | 2 + benchmarks/cases/php-lsp-parity.yaml | 1 + benchmarks/cases/php-precision.yaml | 1 + benchmarks/cases/typescript-baseline.yaml | 1 + benchmarks/cases/typescript-lsp-parity.yaml | 1 + benchmarks/cases/typescript-precision.yaml | 1 + docs/src/content/docs/index.mdx | 71 ++++--- docs/src/content/docs/languages/go.md | 21 +- .../docs/languages/javascript-typescript.md | 16 +- docs/src/content/docs/languages/php.md | 13 +- docs/src/content/docs/methodology.md | 24 ++- docs/src/content/docs/results/index.md | 55 ++--- schema/benchmark-case.schema.json | 10 + src/lib.rs | 101 ++++++++- src/main.rs | 17 ++ src/runners/bifrost.rs | 139 ++++++++++++- src/runners/lsp.rs | 174 +++++++++++++++- src/runners/mod.rs | 193 +++++++++++++++++- src/runners/report_compare.rs | 4 + 24 files changed, 769 insertions(+), 90 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 9bf1c28..36c0692 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -121,6 +121,14 @@ Each case supports both benchmark directions: advertising a narrower, language-specific implementation. It is not a scoring fallback; after the operation is authored, its response is still judged strictly and never substituted with the other operation. + `compatibleOperations` may explicitly list a separately reviewed alternate + endpoint that reaches the same complete authored target. Runners execute it + as a second result: the canonical operation remains the case `status`, while + `requiredDestinationStatus` reports whether the raw canonical or reviewed + compatible results include the destination. That recall-forward field also + tolerates line-only or containing ranges and additional results. It is never + a silent retry and cannot change strict endpoint precision or capability + results. `allowedExtraTargets` may list conservative alternate targets that can accompany the required `expectedDeclaration` but cannot replace it. An alternate may have a different symbol kind when navigation legitimately diff --git a/benchmarks/cases/go-baseline.yaml b/benchmarks/cases/go-baseline.yaml index 2b80af6..2885f27 100644 --- a/benchmarks/cases/go-baseline.yaml +++ b/benchmarks/cases/go-baseline.yaml @@ -177,6 +177,7 @@ cases: displayName: Last usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/example/service_test.go @@ -228,6 +229,7 @@ cases: displayName: DefaultPrefix usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/example/service_test.go @@ -271,6 +273,7 @@ cases: displayName: DefaultRepository usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/example/service_test.go diff --git a/benchmarks/cases/go-lsp-parity.yaml b/benchmarks/cases/go-lsp-parity.yaml index 72bfc63..c0b99d3 100644 --- a/benchmarks/cases/go-lsp-parity.yaml +++ b/benchmarks/cases/go-lsp-parity.yaml @@ -165,6 +165,7 @@ cases: displayName: Last usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/cmd/app/main.go diff --git a/benchmarks/cases/go-precision.yaml b/benchmarks/cases/go-precision.yaml index 2d3b144..dbbf91f 100644 --- a/benchmarks/cases/go-precision.yaml +++ b/benchmarks/cases/go-precision.yaml @@ -75,6 +75,7 @@ cases: displayName: Record usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: { uri: benchmark://source/cmd/app/main.go, range: { start: { line: 10, character: 10 }, end: { line: 10, character: 16 } } } kind: method diff --git a/benchmarks/cases/javascript-baseline.yaml b/benchmarks/cases/javascript-baseline.yaml index 9aefcac..8f1875b 100644 --- a/benchmarks/cases/javascript-baseline.yaml +++ b/benchmarks/cases/javascript-baseline.yaml @@ -258,6 +258,7 @@ cases: displayName: title usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/components.js diff --git a/benchmarks/cases/php-baseline.yaml b/benchmarks/cases/php-baseline.yaml index 24d0705..24d5ff9 100644 --- a/benchmarks/cases/php-baseline.yaml +++ b/benchmarks/cases/php-baseline.yaml @@ -182,6 +182,7 @@ cases: displayName: last usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/Consumer.php @@ -238,6 +239,7 @@ cases: displayName: PREFIX usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/Consumer.php diff --git a/benchmarks/cases/php-lsp-parity.yaml b/benchmarks/cases/php-lsp-parity.yaml index 8070d9f..3be9e1e 100644 --- a/benchmarks/cases/php-lsp-parity.yaml +++ b/benchmarks/cases/php-lsp-parity.yaml @@ -240,6 +240,7 @@ cases: displayName: "$sent" usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/Consumer.php diff --git a/benchmarks/cases/php-precision.yaml b/benchmarks/cases/php-precision.yaml index 4c7ada2..a3c233f 100644 --- a/benchmarks/cases/php-precision.yaml +++ b/benchmarks/cases/php-precision.yaml @@ -21,6 +21,7 @@ cases: displayName: send usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: { uri: benchmark://source/src/Consumer.php, range: { start: { line: 11, character: 15 }, end: { line: 11, character: 19 } } } kind: method diff --git a/benchmarks/cases/typescript-baseline.yaml b/benchmarks/cases/typescript-baseline.yaml index e8b7e85..1ad0a27 100644 --- a/benchmarks/cases/typescript-baseline.yaml +++ b/benchmarks/cases/typescript-baseline.yaml @@ -207,6 +207,7 @@ cases: displayName: name usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/components.tsx diff --git a/benchmarks/cases/typescript-lsp-parity.yaml b/benchmarks/cases/typescript-lsp-parity.yaml index 6451fe5..1804310 100644 --- a/benchmarks/cases/typescript-lsp-parity.yaml +++ b/benchmarks/cases/typescript-lsp-parity.yaml @@ -184,6 +184,7 @@ cases: displayName: name usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: uri: benchmark://source/src/app.ts diff --git a/benchmarks/cases/typescript-precision.yaml b/benchmarks/cases/typescript-precision.yaml index 3b73cab..3ae0071 100644 --- a/benchmarks/cases/typescript-precision.yaml +++ b/benchmarks/cases/typescript-precision.yaml @@ -55,6 +55,7 @@ cases: displayName: Widget usageLookups: - operation: declaration + compatibleOperations: [definition] usage: location: { uri: benchmark://source/src/app.ts, range: { start: { line: 2, character: 14 }, end: { line: 2, character: 20 } } } kind: interface diff --git a/docs/src/content/docs/index.mdx b/docs/src/content/docs/index.mdx index ade80f9..3e9c7bc 100644 --- a/docs/src/content/docs/index.mdx +++ b/docs/src/content/docs/index.mdx @@ -36,14 +36,15 @@ those decisions as executable regression contracts, and the daily workflow runs them against Bifrost `master` so a previously satisfied semantic edge cannot quietly disappear. -The current result starts from the synchronized UsageBench `78e66e5` run and -Bifrost `782522b2`. The Java profile and pinned Bifrost revision were rerun -after correcting JDT workspace isolation and three navigation operations. -Metals was then rerun at 1.6.8; its Scala results were unchanged. The other -eight reference profiles retain their synchronized reports. Every runner -completed without an error. The report shape remains analyzer-neutral: -language servers are the first comparison set, not a permanent limit, and -future analyzers can be added without changing the authored ground truth. +The current result uses UsageBench parent revision `d01a4aa` and pinned Bifrost +`782522b2`. Bifrost, gopls, TypeScript LS, and Intelephense were freshly rerun +after adding the reviewed compatible-operation evidence and report-derived +destination scorer. The corrected Java and refreshed Metals reports are +retained, as are the five other synchronized reference reports whose cases and +operations did not change. Every fresh report records zero runner errors. The +report shape remains analyzer-neutral: language servers are the first +comparison set, not a permanent limit, and future analyzers can be added without +changing the authored ground truth. ## Required destinations found @@ -51,73 +52,81 @@ The headline metric asks the question closest to normal editor use: **did the operation surface every reviewed destination the user needed?** A case counts when all required references are present and every navigation or type lookup includes its expected target. A containing range or additional result does not -turn that into a failure. +turn that into a failure; neither does a line-only location on the expected +line. This is intentionally recall-forward. It does not say that imprecise ranges, unrelated results, or a wrong first choice are harmless; the stricter table -below measures those differences. The denominator contains the **131 cases -scoreable by both Bifrost and the corresponding reference server**. +below measures those differences. The denominator contains the **144 cases +scoreable by both sides using either the canonical operation or an explicitly +reviewed compatible operation**. It does not +silently retry a request: the canonical endpoint remains the source of truth +in the strict table below.
Required destinations found -
+
Bifrost - 90.1% + 88.9%
Reference LSPs - 90.1% + 91.0%
-

118/131 destinations each

+

128/144 · 131/144 destinations

Equal-profile mean -
+
Bifrost - 90.2% + 88.9%
Reference LSPs - 91.4% + 91.6%

equal weight for each of ten profiles

Median paired profile gap - 0.0 points -

the middle cross-profile difference is a tie

+ −3.1 points +

the middle profile favors the reference server

Profile outcomes - 3–3–4 + 3–2–5

Bifrost leads · ties · trails

-| Profile | Shared | Bifrost destinations | Reference destinations | Gap | Reference unsupported | +| Profile | Shared destination-scoreable | Bifrost destinations | Reference destinations | Gap | Reference unavailable | |---|---:|---:|---:|---:|---:| | [C++](languages/cpp/) | 15 | 12/15 (80.0%) | 14/15 (93.3%) | −13.3 pp | 1 | | [C#](languages/csharp/) | 16 | 14/16 (87.5%) | 15/16 (93.8%) | −6.3 pp | 0 | -| [Go](languages/go/) | 6 | 5/6 (83.3%) | 6/6 (100.0%) | −16.7 pp | 6 | +| [Go](languages/go/) | 11 | 9/11 (81.8%) | 11/11 (100.0%) | −18.2 pp | 1 | | [Java](languages/java/) | 11 | 11/11 (100.0%) | 11/11 (100.0%) | 0.0 pp | 0 | -| [JavaScript, TypeScript](languages/javascript-typescript/) | 17 | 15/17 (88.2%) | 15/17 (88.2%) | 0.0 pp | 5 | -| [PHP](languages/php/) | 10 | 10/10 (100.0%) | 10/10 (100.0%) | 0.0 pp | 4 | +| [JavaScript, TypeScript](languages/javascript-typescript/) | 21 | 19/21 (90.5%) | 19/21 (90.5%) | 0.0 pp | 1 | +| [PHP](languages/php/) | 14 | 12/14 (85.7%) | 14/14 (100.0%) | −14.3 pp | 0 | | [Python](languages/python/) | 13 | 11/13 (84.6%) | 13/13 (100.0%) | −15.4 pp | 0 | | [Ruby](languages/ruby/) | 16 | 15/16 (93.8%) | 11/16 (68.8%) | +25.0 pp | 4 | | [Rust](languages/rust/) | 15 | 14/15 (93.3%) | 13/15 (86.7%) | +6.7 pp | 0 | | [Scala](languages/scala/) | 12 | 11/12 (91.7%) | 10/12 (83.3%) | +8.3 pp | 3 | -| **Pooled** | **131** | **118/131 (90.1%)** | **118/131 (90.1%)** | **0.0 pp** | **23** | +| **Pooled** | **144** | **128/144 (88.9%)** | **131/144 (91.0%)** | **−2.1 pp** | **10** | + +Fresh `requiredDestinationStatus` reports reproduce the three affected profile +rows exactly. The pinned full-corpus Bifrost report finds 135/152 individually +scoreable cases; intersecting scoreable case IDs with each reference report +produces the paired 128/144 total shown here. -The two sides reach the same pooled result through different strengths. The -reference servers lead on C++, C#, Go, and Python; Bifrost leads on Ruby, Rust, -and Scala. Java, JavaScript/TypeScript, and PHP tie. On Java specifically, JDT -LS surfaces every required destination even though only four cases satisfy the -stricter range-and-identity contract below. +The reference servers lead on C++, C#, Go, PHP, and Python; Bifrost leads on +Ruby, Rust, and Scala. Java and JavaScript/TypeScript tie. On Java +specifically, JDT LS surfaces every required destination even though only four +cases satisfy the stricter range-and-identity contract below.
“Required destinations found” tolerates result clutter. For example, Ruby diff --git a/docs/src/content/docs/languages/go.md b/docs/src/content/docs/languages/go.md index 1daf761..eee7ef7 100644 --- a/docs/src/content/docs/languages/go.md +++ b/docs/src/content/docs/languages/go.md @@ -3,16 +3,18 @@ title: Go — Bifrost and gopls description: Compare package, embedding, receiver, and interface-method reference behavior. --- -| Runner | Exact | Position unverified | Hard | Unsupported | -|---|---:|---:|---:|---:| -| Bifrost | 9 | 0 | 2 | 1 | -| gopls | 6 | 0 | 0 | 6 | +| Runner | Required destinations found | Strict exact | Position unverified | Hard | Strict unsupported | +|---|---:|---:|---:|---:|---:| +| Bifrost | 9/11 | 9 | 0 | 2 | 1 | +| gopls | 11/11 | 6 | 0 | 0 | 6 | ## Strong agreement -Both analyzers can score six common cases and are exact together on five. -gopls alone is exact on the dot-import concrete-receiver call; Bifrost currently -misclassifies the imported `Record` selectors as shadowed local bindings. +In the compatibility-aware editor view, eleven cases are destination-scoreable +by both analyzers. gopls reaches all eleven destinations; Bifrost misses the +dot-import concrete-receiver call and the interface-family reference set. The +strict canonical comparison still has only six shared cases, where gopls is +exact on all six and Bifrost is exact on five. ## The interface-family split @@ -22,6 +24,11 @@ Definition. Bifrost can score five of those six, but its interface receiver case still misses the two conservative concrete candidates and the declaration lookup. +Five of those canonical Declaration contracts also record reviewed Definition +compatibility. They raise the shared destination-scoreable denominator from six +to eleven, but do not change the strict gopls Declaration denominator or +capability result. + This consistent behavior may reflect an intentional method-family policy. It is not enough to claim object insensitivity: the case does not vary allocation or receiver contexts while keeping every other factor fixed. diff --git a/docs/src/content/docs/languages/javascript-typescript.md b/docs/src/content/docs/languages/javascript-typescript.md index 1cd45b7..c7f1e5a 100644 --- a/docs/src/content/docs/languages/javascript-typescript.md +++ b/docs/src/content/docs/languages/javascript-typescript.md @@ -3,10 +3,10 @@ title: JavaScript and TypeScript — Bifrost and TypeScript LS description: Compare ES modules, CommonJS, barrels, types, JSX, and conservative receiver evidence. --- -| Runner | Exact | Position unverified | Hard | Unsupported | Not planned | -|---|---:|---:|---:|---:|---:| -| Bifrost | 20 | 0 | 2 | 0 | 1 | -| TypeScript LS | 13 | 2 | 2 | 5 | 1 | +| Runner | Required destinations found | Strict exact | Position unverified | Hard | Strict unsupported | Not planned | +|---|---:|---:|---:|---:|---:|---:| +| Bifrost | 19/21 | 20 | 0 | 2 | 0 | 1 | +| TypeScript LS | 19/21 | 13 | 2 | 2 | 5 | 1 | ## TypeScript agreement after import policy @@ -15,6 +15,14 @@ policy near misses no longer obscure semantic agreement. On the 17 cases both sides can score, 11 are exact for both, 4 only for Bifrost, and 2 only for the TypeScript language server. +The class-property contract, the two TypeScript-property contracts, and the +type-annotation contract retain their reviewed Declaration operation while +recording Definition as explicit compatibility evidence. Those four cases raise +the shared destination-scoreable denominator from 17 to 21 without rewriting +the strict endpoint comparison. The CommonJS exported-member contract is +intentionally excluded: its Definition target is a distinct +function-expression token. + ## CommonJS split Bifrost satisfies the destructured-function and barrel-class CommonJS cases diff --git a/docs/src/content/docs/languages/php.md b/docs/src/content/docs/languages/php.md index 6c3b07f..0bac985 100644 --- a/docs/src/content/docs/languages/php.md +++ b/docs/src/content/docs/languages/php.md @@ -3,10 +3,10 @@ title: PHP — Bifrost and Intelephense description: Compare namespaces, imports, traits, interfaces, receivers, properties, and static members. --- -| Runner | Exact | Position unverified | Hard | Unsupported | -|---|---:|---:|---:|---:| -| Bifrost | 12 | 0 | 2 | 0 | -| Intelephense | 9 | 0 | 1 | 4 | +| Runner | Required destinations found | Strict exact | Position unverified | Hard | Strict unsupported | +|---|---:|---:|---:|---:|---:| +| Bifrost | 12/14 | 12 | 0 | 2 | 0 | +| Intelephense | 14/14 | 9 | 0 | 1 | 4 | ## Shared strengths @@ -14,6 +14,11 @@ Both analyzers can score ten common cases and are exact together on nine. Bifrost alone is exact on class construction because Intelephense returns both the class and explicit constructor as navigation targets. +Four canonical Declaration contracts—properties, a constant, and a bodyless +interface member—also record Definition as reviewed compatibility evidence. +They raise the shared destination-scoreable denominator from ten to fourteen; +strict operation precision and capability counts remain unchanged. + ## Bifrost recall edge Bifrost also scores four Declaration-oriented cases that Intelephense does not diff --git a/docs/src/content/docs/methodology.md b/docs/src/content/docs/methodology.md index 0bb65b0..0eaaf0a 100644 --- a/docs/src/content/docs/methodology.md +++ b/docs/src/content/docs/methodology.md @@ -19,11 +19,14 @@ The user-facing metric counts a case when: 2. every navigation lookup includes its expected destination; and 3. every type lookup includes its expected type destination. -A broader containing range and additional returned results are tolerated. This -models the basic editor question—“did the operation take me to, or list, the -code I needed?”—without pretending that range precision and result clutter are -invisible to users. Unsupported operations remain outside the shared -denominator. +A line-only location, a broader containing range, and additional returned +results are tolerated. This models the basic editor question—“did the operation +take me to, or list, the code I needed?”—without pretending that range +precision and result clutter are invisible to users. A case remains in this +shared denominator when both analyzers can execute every required lookup +through either the canonical operation or an explicitly reviewed compatible +operation. Operations unavailable through both paths remain outside it; the +strict metric continues to use only the authored canonical operation. This is best read as **required-destination recall**, not an all-purpose quality score. A response containing the expected target among many unrelated targets @@ -72,6 +75,17 @@ capability-advertisement bias. It does **not** make the scorer lenient: once a case selects Declaration or Definition, that exact request must satisfy the strict singleton target contract, with no fallback to the other endpoint. +Cases may additionally declare reviewed `compatibleOperations` when another +endpoint reaches the exact same authored target. The runner records these as +separate alternate results: `status` remains the canonical protocol-specific +score, while `requiredDestinationStatus` is computed from the raw canonical +and reviewed-compatible locations. It records `found` when all expected +destinations are present even if ranges contain the expected token or the +response includes extras. An alternate is never queried silently, never merged +into the canonical response, and cannot improve the strict endpoint result. +Per-report `totals.requiredDestinations` makes this metric machine-readable +without changing the canonical counters. + The strict result is not labeled generic LSP compliance. The current [`Location`](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.18/specification/#location) shape requires a URI and range, and the protocol defines how ranges are diff --git a/docs/src/content/docs/results/index.md b/docs/src/content/docs/results/index.md index 77af295..9369ca8 100644 --- a/docs/src/content/docs/results/index.md +++ b/docs/src/content/docs/results/index.md @@ -8,50 +8,59 @@ description: Required-destination and strict-contract results from the reviewed > `legacy_unattributed`. A second independent review, preregistration, and an > immutable freeze are still required for evaluation promotion. -This corrected native result replaces the legacy 16 July figures. Eight -unchanged reference profiles retain their synchronized 24 July reports. Java -was rerun with the same pinned JDT LS after correcting its workspace path and -changing three ordinary source-navigation lookups from Declaration to -Definition. Metals was rerun at 1.6.8 with unchanged Scala case outcomes. -Bifrost was rerun across the full corpus at the same pinned revision; its -totals were unchanged. All runner processes completed without errors. +This corrected native result replaces the legacy 16 July figures. Bifrost was +rerun across the full corpus at the pinned revision. gopls, TypeScript LS, and +Intelephense were rerun after adding reviewed compatible-operation evidence and +the machine-readable destination scorer. Java had already been rerun with the +same pinned JDT LS after correcting its workspace path and ordinary +source-navigation operations; Metals had been refreshed at 1.6.8. The other +five reference profiles retain their synchronized 24 July reports because +their cases and operations did not change. All fresh reports record zero runner +errors. | Run fact | Value | |---|---| | Date | 24 July 2026 | -| Base synchronized UsageBench revision | `78e66e5dd3589d4543f1b19a8b3566fa9afd644a` | -| Corrections | Java workspace isolation and Definition authoring; Metals 1.6.8 refresh | +| UsageBench parent revision | `d01a4aa60b4516529a63009ef312c6f34d21b80a` | +| Corrections | Java workspace isolation and Definition authoring; Metals 1.6.8 refresh; 13 reviewed compatible-operation annotations and report-derived destination scoring | | Bifrost revision | `782522b245fc86e3d39b1cdc0488553a1d262212` (pinned synchronized revision) | | Host | macOS arm64, native and host-specific | | Scoring | Headline required-destination recall; secondary strict range and identity conformance | ## Required destinations found -The primary comparison uses the 131 cases whose authored operation is scoreable -by both Bifrost and the corresponding reference server. It asks whether every -reviewed reference was returned and every navigation or type lookup included -its expected destination. Broader containing ranges and additional results are +The primary comparison uses the 144 cases scoreable by both Bifrost and the +corresponding reference server through the canonical operation or an explicitly +reviewed compatible operation. It asks whether every reviewed reference was +returned and every navigation or type lookup included its expected +destination. Line-only or broader containing ranges and additional results are tolerated in this view because an editor user can still reach the required -code. +code. Compatible operations are reported separately; they do not alter the +strict canonical results below. -| Language | Shared | Bifrost destinations | Reference destinations | +| Language | Shared destination-scoreable | Bifrost destinations | Reference destinations | |---|---:|---:|---:| | C++ | 15 | 12/15 (80.0%) | 14/15 (93.3%) | | C# | 16 | 14/16 (87.5%) | 15/16 (93.8%) | -| Go | 6 | 5/6 (83.3%) | 6/6 (100.0%) | +| Go | 11 | 9/11 (81.8%) | 11/11 (100.0%) | | Java | 11 | 11/11 (100.0%) | 11/11 (100.0%) | -| JavaScript, TypeScript | 17 | 15/17 (88.2%) | 15/17 (88.2%) | -| PHP | 10 | 10/10 (100.0%) | 10/10 (100.0%) | +| JavaScript, TypeScript | 21 | 19/21 (90.5%) | 19/21 (90.5%) | +| PHP | 14 | 12/14 (85.7%) | 14/14 (100.0%) | | Python | 13 | 11/13 (84.6%) | 13/13 (100.0%) | | Ruby | 16 | 15/16 (93.8%) | 11/16 (68.8%) | | Rust | 15 | 14/15 (93.3%) | 13/15 (86.7%) | | Scala | 12 | 11/12 (91.7%) | 10/12 (83.3%) | -| **Pooled** | **131** | **118/131 (90.1%)** | **118/131 (90.1%)** | +| **Pooled** | **144** | **128/144 (88.9%)** | **131/144 (91.0%)** | -The pooled destination result is a tie. With each profile weighted equally, -Bifrost averages **90.2%** and the reference servers **91.4%**; the median -paired profile difference is **0.0 percentage points**. Bifrost leads three -profiles, ties three, and trails four. +The fresh report fields reproduce the affected rows directly: Go pairs to +9/11 versus 11/11, JavaScript/TypeScript to 19/21 versus 19/21, and PHP to +12/14 versus 14/14. Bifrost's full report finds 135 of its 152 individually +scoreable cases; intersecting scoreable case IDs with each reference profile +produces the 128/144 paired total above. + +With each profile weighted equally, Bifrost averages **88.9%** and the +reference servers **91.6%**; the median paired profile difference is **−3.1 +percentage points**. Bifrost leads three profiles, ties two, and trails five. This is a recall-forward measure, not a declaration that all returned results are equally good. A case can surface every required destination while also diff --git a/schema/benchmark-case.schema.json b/schema/benchmark-case.schema.json index af4e096..c22182f 100644 --- a/schema/benchmark-case.schema.json +++ b/schema/benchmark-case.schema.json @@ -276,6 +276,16 @@ "enum": ["profile_default", "declaration", "definition"], "default": "profile_default" }, + "compatibleOperations": { + "description": "Explicitly reviewed alternate operations that reach the same authored target. These are reported separately for compatibility/user-felt navigation and never replace the canonical `operation` score.", + "type": "array", + "default": [], + "uniqueItems": true, + "items": { + "type": "string", + "enum": ["declaration", "definition"] + } + }, "expectNoMovement": { "type": "boolean", "default": false diff --git a/src/lib.rs b/src/lib.rs index f9bd070..da16822 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -182,6 +182,11 @@ pub enum Disambiguation { pub struct UsageLookup { #[serde(default)] pub operation: NavigationOperation, + /// Explicitly reviewed alternate operations that reach the same authored + /// target. These are reported separately for compatibility/user-felt + /// navigation and never replace the canonical `operation` score. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub compatible_operations: Vec, /// Accept either no result or an exact self-target, but reject navigation /// to any other token. `expectedDeclaration` must equal `usage`. #[serde(default)] @@ -194,7 +199,9 @@ pub struct UsageLookup { pub allowed_extra_targets: Vec, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] #[serde(rename_all = "snake_case")] pub enum NavigationOperation { /// Preserve the legacy profile-selected endpoint while development cases @@ -568,6 +575,39 @@ impl BenchmarkCase { self.id ); } + if lookup + .compatible_operations + .iter() + .any(|operation| *operation == NavigationOperation::ProfileDefault) + { + bail!( + "case {} usageLookups compatibleOperations must select an explicit operation", + self.id + ); + } + if lookup + .compatible_operations + .iter() + .any(|operation| *operation == lookup.operation) + { + bail!( + "case {} usageLookups compatibleOperations must differ from operation", + self.id + ); + } + let compatible_operation_count = lookup.compatible_operations.len(); + let distinct_compatible_operation_count = lookup + .compatible_operations + .iter() + .copied() + .collect::>() + .len(); + if compatible_operation_count != distinct_compatible_operation_count { + bail!( + "case {} usageLookups compatibleOperations must not contain duplicates", + self.id + ); + } } for lookup in &self.type_lookups { @@ -880,6 +920,37 @@ cases: [] schema["properties"]["referencePolicy"]["enum"], serde_json::json!(["external_usages", "bindings_optional", "bindings_required"]) ); + assert_eq!( + schema["$defs"]["usageLookup"]["properties"]["compatibleOperations"]["items"]["enum"], + serde_json::json!(["declaration", "definition"]) + ); + assert_eq!( + schema["$defs"]["usageLookup"]["properties"]["compatibleOperations"]["uniqueItems"], + true + ); + } + + #[test] + fn validation_rejects_profile_default_compatible_operation() { + let mut case = compatible_operation_case(); + case.usage_lookups[0].compatible_operations = vec![NavigationOperation::ProfileDefault]; + + let error = case.validate(None, PositionEncoding::Utf16).unwrap_err(); + + assert!(format!("{error:#}").contains("must select an explicit operation")); + } + + #[test] + fn validation_rejects_duplicate_compatible_operations() { + let mut case = compatible_operation_case(); + case.usage_lookups[0].compatible_operations = vec![ + NavigationOperation::Definition, + NavigationOperation::Definition, + ]; + + let error = case.validate(None, PositionEncoding::Utf16).unwrap_err(); + + assert!(format!("{error:#}").contains("must not contain duplicates")); } #[test] @@ -1419,4 +1490,32 @@ cases: [] )) .unwrap() } + + fn compatible_operation_case() -> BenchmarkCase { + serde_yaml::from_str( + r#" +id: compatible-operation +usageLookups: + - operation: declaration + compatibleOperations: [definition] + usage: + location: + uri: benchmark://source/src/lib.rs + range: + start: { line: 1, character: 4 } + end: { line: 1, character: 8 } + kind: function + displayName: item + expectedDeclaration: + location: + uri: benchmark://source/src/lib.rs + range: + start: { line: 0, character: 4 } + end: { line: 0, character: 8 } + kind: function + displayName: item +"#, + ) + .unwrap() + } } diff --git a/src/main.rs b/src/main.rs index 0f3dc93..2399c40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -173,6 +173,7 @@ fn main() -> Result<()> { report.totals.skipped, report.totals.errors ); + print_required_destination_totals(&report); print_run_details(&report); if report.totals.failed > 0 || report.totals.position_unverified > 0 @@ -220,6 +221,7 @@ fn main() -> Result<()> { report.totals.skipped, report.totals.errors ); + print_required_destination_totals(&report); print_run_details(&report); if report.totals.failed > 0 || report.totals.position_unverified > 0 @@ -238,6 +240,21 @@ fn main() -> Result<()> { Ok(()) } +fn print_required_destination_totals(report: &BifrostRunReport) { + let totals = &report.totals.required_destinations; + println!( + "required destinations: {}/{} found ({} missing), {} not planned, {} unsupported, {} skipped, {} error(s), {} unreported", + totals.found, + totals.scoreable_cases, + totals.missing, + totals.not_planned, + totals.unsupported, + totals.skipped, + totals.errors, + totals.unreported, + ); +} + fn print_run_details(report: &BifrostRunReport) { for document in &report.documents { for case in &document.cases { diff --git a/src/runners/bifrost.rs b/src/runners/bifrost.rs index 25e4326..97987f5 100644 --- a/src/runners/bifrost.rs +++ b/src/runners/bifrost.rs @@ -1,13 +1,14 @@ use super::mcp::{McpSession, ToolClient as SearchToolsClient}; use super::{ combine_case_status, compute_totals, location_match, normalize_symbol_location, path_to_slash, - resolve_usagebench_provenance, score_declaration_locations, score_navigation_response, - symbol_kind_name, CapabilitySupport, LocationMatch, RunInvocation, RunReport, RunnerCapability, - RunnerMetadata, RunnerOperation, + required_destination_status, resolve_usagebench_provenance, score_declaration_locations, + score_navigation_response, symbol_kind_name, CapabilitySupport, LocationMatch, RunInvocation, + RunReport, RunnerCapability, RunnerMetadata, RunnerOperation, }; pub use super::{ - CaseRunReport, CaseStatus, DeclarationUsageReport, DocumentRunReport, NormalizedLocation, - RunDiagnostic, RunTotals, TypeLookupReport, UsageDefinitionReport, + CaseRunReport, CaseStatus, CompatibleUsageDefinitionReport, DeclarationUsageReport, + DocumentRunReport, NormalizedLocation, RequiredDestinationStatus, RunDiagnostic, RunTotals, + TypeLookupReport, UsageDefinitionReport, }; use crate::{ benchmark_source_path, find_repo_root_for_path, BenchmarkCase, BenchmarkDocument, Location, @@ -302,6 +303,8 @@ fn run_case( unsupported_reason: Some(unsupported.reason.clone()), declaration_to_usages: None, usage_to_declaration: Vec::new(), + compatible_usage_to_declaration: Vec::new(), + required_destination_status: Some(RequiredDestinationStatus::Unsupported), type_lookups: Vec::new(), diagnostics: Vec::new(), }; @@ -331,6 +334,31 @@ fn run_case( .map(skipped_definition_lookup) .collect::>() }; + let compatible_usage_to_declaration = case + .usage_lookups + .iter() + .enumerate() + .filter(|(_, lookup)| !lookup.compatible_operations.is_empty()) + .map( + |(usage_lookup_index, lookup)| CompatibleUsageDefinitionReport { + usage_lookup_index, + canonical_operation: lookup.operation, + reports: lookup + .compatible_operations + .iter() + .map(|operation| { + let mut compatible_lookup = lookup.clone(); + compatible_lookup.operation = *operation; + if include_definition_lookups { + run_usage_to_declaration(&compatible_lookup, encoding, session) + } else { + skipped_definition_lookup(&compatible_lookup) + } + }) + .collect(), + }, + ) + .collect::>(); let type_lookups = if include_definition_lookups { case.type_lookups .iter() @@ -362,6 +390,17 @@ fn run_case( not_planned_reason.as_deref(), &mut diagnostics, ); + let mut required_destination_status = required_destination_status( + declaration_to_usages.as_ref(), + &usage_to_declaration, + &compatible_usage_to_declaration, + &type_lookups, + ); + if required_destination_status != RequiredDestinationStatus::Error + && not_planned_reason.is_some() + { + required_destination_status = RequiredDestinationStatus::NotPlanned; + } CaseRunReport { id: case.id.clone(), status, @@ -373,6 +412,8 @@ fn run_case( .map(|unsupported| unsupported.reason.clone()), declaration_to_usages, usage_to_declaration, + compatible_usage_to_declaration, + required_destination_status: Some(required_destination_status), type_lookups, diagnostics, } @@ -1944,6 +1985,11 @@ mod tests { unsupported: 0, skipped: 0, errors: 0, + required_destinations: crate::runners::RequiredDestinationTotals { + scoreable_cases: 1, + found: 1, + ..Default::default() + }, }, documents: vec![DocumentRunReport { case_file: "benchmarks/cases/rust.yaml".to_string(), @@ -1981,6 +2027,8 @@ mod tests { raw_statuses: vec!["ok".to_string()], }), usage_to_declaration: Vec::new(), + compatible_usage_to_declaration: Vec::new(), + required_destination_status: Some(RequiredDestinationStatus::Found), type_lookups: Vec::new(), diagnostics: Vec::new(), }], @@ -2004,6 +2052,11 @@ mod tests { "c".repeat(64) ); assert_eq!(json["totals"]["passed"], 1); + assert_eq!(json["totals"]["requiredDestinations"]["found"], 1); + assert_eq!( + json["documents"][0]["cases"][0]["requiredDestinationStatus"], + "found" + ); assert_eq!( json["documents"][0]["cases"][0]["declarationToUsages"]["unexpected"][0]["path"], "src/extra.rs" @@ -2047,6 +2100,59 @@ mod tests { assert_eq!(client.calls[1].1["include_bindings"], true); } + #[test] + fn compatible_navigation_is_reported_without_replacing_canonical_status() { + let mut case = benchmark_case(); + case.usage_lookups[0].operation = crate::NavigationOperation::Declaration; + case.usage_lookups[0].compatible_operations = vec![crate::NavigationOperation::Definition]; + let mut client = MockClient::new(vec![ + tool( + "search_symbols", + search_symbols_json("src/service.rs", "example.build_service", 30), + ), + tool( + "scan_usages_by_location", + scan_usages_json(vec![("src/lib.rs", 8)], false), + ), + tool( + GET_DECLARATIONS_BY_LOCATION_TOOL, + get_definitions_by_location_json("no_declaration", Vec::new()), + ), + tool( + GET_DEFINITIONS_BY_LOCATION_TOOL, + get_definitions_by_location_json("resolved", vec![("src/service.rs", 30)]), + ), + ]); + + let report = run_case( + &case, + PositionEncoding::Utf16, + ReferencePolicy::BindingsOptional, + None, + &mut client, + false, + true, + ); + + assert_eq!(report.status, CaseStatus::Failed); + assert_eq!( + report.required_destination_status, + Some(RequiredDestinationStatus::Found) + ); + assert_eq!( + report.usage_to_declaration[0].operation, + NavigationOperation::Declaration + ); + assert_eq!( + report.compatible_usage_to_declaration[0].reports[0].operation, + NavigationOperation::Definition + ); + assert_eq!( + report.compatible_usage_to_declaration[0].usage_lookup_index, + 0 + ); + } + #[test] fn external_usage_policy_excludes_bindings_but_keeps_same_owner_references() { let case = benchmark_case(); @@ -2351,6 +2457,10 @@ mod tests { ); assert_eq!(report.status, CaseStatus::PositionUnverified); + assert_eq!( + report.required_destination_status, + Some(RequiredDestinationStatus::Found) + ); let declaration = report.declaration_to_usages.unwrap(); assert!(declaration.actual.is_empty()); assert_eq!(declaration.unproven.len(), 2); @@ -2434,6 +2544,10 @@ mod tests { ); assert_eq!(report.status, CaseStatus::Failed); + assert_eq!( + report.required_destination_status, + Some(RequiredDestinationStatus::Found) + ); let declaration = report.declaration_to_usages.unwrap(); assert_eq!(declaration.status, CaseStatus::Failed); assert_eq!(declaration.unexpected.len(), 1); @@ -2468,6 +2582,10 @@ mod tests { ); assert_eq!(report.status, CaseStatus::Failed); + assert_eq!( + report.required_destination_status, + Some(RequiredDestinationStatus::Missing) + ); let declaration = report.declaration_to_usages.unwrap(); assert_eq!(declaration.status, CaseStatus::Failed); assert_eq!(declaration.missing.len(), 1); @@ -2912,6 +3030,10 @@ mod tests { "scan_usages_by_location", scan_usages_json(vec![("src/lib.rs", 8)], true), ), + tool( + GET_DEFINITIONS_BY_LOCATION_TOOL, + get_definitions_by_location_json("resolved", vec![("src/service.rs", 30)]), + ), ]); let report = run_case( @@ -2921,10 +3043,14 @@ mod tests { None, &mut client, false, - false, + true, ); assert_eq!(report.status, CaseStatus::Failed); + assert_eq!( + report.required_destination_status, + Some(RequiredDestinationStatus::Found) + ); let declaration = report.declaration_to_usages.unwrap(); assert!(declaration.partial); assert!(declaration.raw_statuses.contains(&"partial".to_string())); @@ -3457,6 +3583,7 @@ mod tests { allowed_unproven_usages: Vec::new(), usage_lookups: vec![UsageLookup { operation: crate::NavigationOperation::ProfileDefault, + compatible_operations: Vec::new(), expect_no_movement: false, usage: symbol_location("src/lib.rs", 7, 18, "build_service", SymbolKind::Function), expected_declaration: symbol_location( diff --git a/src/runners/lsp.rs b/src/runners/lsp.rs index d3b6b44..e7e22ff 100644 --- a/src/runners/lsp.rs +++ b/src/runners/lsp.rs @@ -2,10 +2,11 @@ use super::bifrost::{command_output_with_timeout, prepare_source_root}; use super::lsp_protocol::{InitializeResult, LspSession}; use super::{ combine_case_status, compute_totals, location_match, navigation_response_status, - normalize_symbol_location, path_to_slash, score_declaration_locations, - score_navigation_response, symbol_kind_name, CapabilitySupport, CaseRunReport, CaseStatus, - ClassifiedExtraUsage, DeclarationUsageReport, DocumentRunReport, ExtraUsageClassification, - ExtraUsageDisposition, LocationMatch, NormalizedLocation, RunDiagnostic, RunInvocation, + normalize_symbol_location, path_to_slash, required_destination_status, + score_declaration_locations, score_navigation_response, symbol_kind_name, CapabilitySupport, + CaseRunReport, CaseStatus, ClassifiedExtraUsage, CompatibleUsageDefinitionReport, + DeclarationUsageReport, DocumentRunReport, ExtraUsageClassification, ExtraUsageDisposition, + LocationMatch, NormalizedLocation, RequiredDestinationStatus, RunDiagnostic, RunInvocation, RunReport, RunTotals, RunnerCapability, RunnerMetadata, RunnerOperation, TypeLookupReport, UsageDefinitionReport, }; @@ -371,6 +372,8 @@ fn run_case( unsupported_reason: Some(unsupported.reason.clone()), declaration_to_usages: None, usage_to_declaration: Vec::new(), + compatible_usage_to_declaration: Vec::new(), + required_destination_status: Some(RequiredDestinationStatus::Unsupported), type_lookups: Vec::new(), diagnostics: Vec::new(), }; @@ -397,6 +400,33 @@ fn run_case( ) }) .collect::>(); + let compatible_usage_to_declaration = case + .usage_lookups + .iter() + .enumerate() + .filter(|(_, lookup)| !lookup.compatible_operations.is_empty()) + .map( + |(usage_lookup_index, lookup)| CompatibleUsageDefinitionReport { + usage_lookup_index, + canonical_operation: lookup.operation, + reports: lookup + .compatible_operations + .iter() + .map(|operation| { + let mut compatible_lookup = lookup.clone(); + compatible_lookup.operation = *operation; + run_definition( + &compatible_lookup, + context.profile, + context.source_root, + context.capabilities, + session, + ) + }) + .collect(), + }, + ) + .collect::>(); let type_lookups = case .type_lookups .iter() @@ -422,6 +452,16 @@ fn run_case( if status != CaseStatus::Error && case.not_planned.is_some() { status = CaseStatus::NotPlanned; } + let mut required_destination_status = required_destination_status( + declaration_to_usages.as_ref(), + &usage_to_declaration, + &compatible_usage_to_declaration, + &type_lookups, + ); + if required_destination_status != RequiredDestinationStatus::Error && case.not_planned.is_some() + { + required_destination_status = RequiredDestinationStatus::NotPlanned; + } CaseRunReport { id: case.id.clone(), status, @@ -430,6 +470,8 @@ fn run_case( unsupported_reason: case.unsupported.as_ref().map(|item| item.reason.clone()), declaration_to_usages, usage_to_declaration, + compatible_usage_to_declaration, + required_destination_status: Some(required_destination_status), type_lookups, diagnostics, } @@ -1353,6 +1395,8 @@ fn error_case(case: &BenchmarkCase, kind: &str, message: &str) -> CaseRunReport unsupported_reason: case.unsupported.as_ref().map(|item| item.reason.clone()), declaration_to_usages: None, usage_to_declaration: Vec::new(), + compatible_usage_to_declaration: Vec::new(), + required_destination_status: Some(RequiredDestinationStatus::Error), type_lookups: Vec::new(), diagnostics: vec![RunDiagnostic { kind: kind.to_string(), @@ -1505,6 +1549,112 @@ mod tests { ); } + #[test] + fn compatible_definition_is_destination_evidence_not_a_strict_declaration_fallback() { + let usage = test_location("src/main.go", 8); + let expected = test_location("src/service.go", 3); + let canonical = UsageDefinitionReport { + status: CaseStatus::Unsupported, + operation: NavigationOperation::Declaration, + usage: usage.clone(), + expected_declaration: expected.clone(), + actual_declarations: Vec::new(), + raw_status: "declaration_not_advertised".to_string(), + diagnostics: Vec::new(), + }; + let compatible = UsageDefinitionReport { + status: CaseStatus::Passed, + operation: NavigationOperation::Definition, + usage, + expected_declaration: expected.clone(), + actual_declarations: vec![expected], + raw_status: "ok".to_string(), + diagnostics: Vec::new(), + }; + let compatible_reports = vec![CompatibleUsageDefinitionReport { + usage_lookup_index: 0, + canonical_operation: NavigationOperation::Declaration, + reports: vec![compatible], + }]; + + let strict_status = combine_case_status(None, std::slice::from_ref(&canonical), &[]); + let destination_status = required_destination_status( + None, + std::slice::from_ref(&canonical), + &compatible_reports, + &[], + ); + assert_eq!(strict_status, CaseStatus::Unsupported); + assert_eq!(destination_status, RequiredDestinationStatus::Found); + + let documents = vec![DocumentRunReport { + case_file: "benchmarks/cases/go-baseline.yaml".to_string(), + language: "go".to_string(), + source_root: "/tmp/source".to_string(), + corpus_partition: crate::CorpusPartition::Development, + corpus_selection: crate::CorpusSelection::AnalyzerInformed, + ground_truth_status: crate::GroundTruthReviewStatus::LegacyUnattributed, + reference_policy: ReferencePolicy::BindingsOptional, + cases: vec![CaseRunReport { + id: "go-compatible-navigation".to_string(), + status: strict_status, + required_destination_status: Some(destination_status), + expected_failure_reason: None, + not_planned_reason: None, + unsupported_reason: None, + declaration_to_usages: None, + usage_to_declaration: vec![canonical], + compatible_usage_to_declaration: compatible_reports, + type_lookups: Vec::new(), + diagnostics: Vec::new(), + }], + }]; + + let totals = compute_totals(&documents); + assert_eq!(totals.cases, 0); + assert_eq!(totals.unsupported, 1); + assert_eq!(totals.required_destinations.scoreable_cases, 1); + assert_eq!(totals.required_destinations.found, 1); + } + + #[test] + fn unavailable_compatible_route_does_not_hide_a_scoreable_canonical_miss() { + let usage = test_location("src/main.go", 8); + let expected = test_location("src/service.go", 3); + let canonical = UsageDefinitionReport { + status: CaseStatus::Failed, + operation: NavigationOperation::Declaration, + usage: usage.clone(), + expected_declaration: expected.clone(), + actual_declarations: Vec::new(), + raw_status: "no_declaration".to_string(), + diagnostics: Vec::new(), + }; + let unavailable_alternate = UsageDefinitionReport { + status: CaseStatus::Unsupported, + operation: NavigationOperation::Definition, + usage, + expected_declaration: expected, + actual_declarations: Vec::new(), + raw_status: "definition_not_advertised".to_string(), + diagnostics: Vec::new(), + }; + + assert_eq!( + required_destination_status( + None, + std::slice::from_ref(&canonical), + &[CompatibleUsageDefinitionReport { + usage_lookup_index: 0, + canonical_operation: NavigationOperation::Declaration, + reports: vec![unavailable_alternate], + }], + &[], + ), + RequiredDestinationStatus::Missing + ); + } + #[test] fn parses_locations_and_location_links() { let root = Path::new("/tmp/workspace"); @@ -1598,13 +1748,27 @@ mod tests { assert_eq!( score_navigation_response( - &[expected.clone(), unexpected], + &[expected.clone(), unexpected.clone()], &expected, std::slice::from_ref(&allowed), false, ), (CaseStatus::Failed, "multiple_targets") ); + + let report = UsageDefinitionReport { + status: CaseStatus::Failed, + operation: NavigationOperation::Definition, + usage: test_location("src/main.py", 3), + expected_declaration: expected.clone(), + actual_declarations: vec![expected, unexpected], + raw_status: "multiple_targets".to_string(), + diagnostics: Vec::new(), + }; + assert_eq!( + required_destination_status(None, std::slice::from_ref(&report), &[], &[]), + RequiredDestinationStatus::Found + ); } #[test] diff --git a/src/runners/mod.rs b/src/runners/mod.rs index 992ccf5..05e7db2 100644 --- a/src/runners/mod.rs +++ b/src/runners/mod.rs @@ -214,6 +214,27 @@ pub struct RunTotals { pub unsupported: usize, pub skipped: usize, pub errors: usize, + /// Recall-forward result derived from the raw locations in each case. + /// Unlike the strict counters above, this tolerates line-only or containing + /// ranges and extra results, and may use an explicitly reviewed compatible + /// operation. + #[serde(default)] + pub required_destinations: RequiredDestinationTotals, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RequiredDestinationTotals { + /// Cases for which every required lookup was available and executed. + pub scoreable_cases: usize, + pub found: usize, + pub missing: usize, + pub not_planned: usize, + pub unsupported: usize, + pub skipped: usize, + pub errors: usize, + /// Reports produced before this metric was added. + pub unreported: usize, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -244,11 +265,28 @@ pub enum CaseStatus { Error, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RequiredDestinationStatus { + Found, + Missing, + NotPlanned, + Unsupported, + Skipped, + Error, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct CaseRunReport { pub id: String, pub status: CaseStatus, + /// Recall-forward destination result. This is computed from raw locations, + /// tolerates line-only or containing ranges and extra results, and may use + /// an explicitly reviewed compatible operation. `status` remains the strict + /// canonical endpoint score. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required_destination_status: Option, #[serde(skip_serializing_if = "Option::is_none")] pub expected_failure_reason: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -259,6 +297,11 @@ pub struct CaseRunReport { pub declaration_to_usages: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub usage_to_declaration: Vec, + /// Alternate endpoint results. Each entry identifies its canonical + /// `usageToDeclaration` lookup and is compatibility evidence, not a + /// silent retry of the canonical score. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub compatible_usage_to_declaration: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub type_lookups: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -333,6 +376,14 @@ pub struct UsageDefinitionReport { pub diagnostics: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct CompatibleUsageDefinitionReport { + pub usage_lookup_index: usize, + pub canonical_operation: NavigationOperation, + pub reports: Vec, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct TypeLookupReport { @@ -556,11 +607,124 @@ pub(crate) fn combine_case_status( definitions: &[UsageDefinitionReport], types: &[TypeLookupReport], ) -> CaseStatus { - let statuses = declaration + combine_case_status_from_statuses( + declaration + .into_iter() + .map(|report| report.status) + .chain(definitions.iter().map(|report| report.status)), + types.iter().map(|report| report.status), + ) +} + +pub(crate) fn required_destination_status( + declaration: Option<&DeclarationUsageReport>, + canonical_definitions: &[UsageDefinitionReport], + compatible_definitions: &[CompatibleUsageDefinitionReport], + types: &[TypeLookupReport], +) -> RequiredDestinationStatus { + let mut components = Vec::new(); + if let Some(report) = declaration { + components.push(declaration_destination_status(report)); + } + for (index, canonical) in canonical_definitions.iter().enumerate() { + let alternatives = compatible_definitions + .iter() + .filter(|compatible| compatible.usage_lookup_index == index) + .flat_map(|compatible| compatible.reports.iter()); + components.push(best_navigation_destination_status( + std::iter::once(canonical).chain(alternatives), + )); + } + components.extend(types.iter().map(type_destination_status)); + combine_required_destination_statuses(components) +} + +fn declaration_destination_status(report: &DeclarationUsageReport) -> RequiredDestinationStatus { + match report.status { + CaseStatus::Error => RequiredDestinationStatus::Error, + CaseStatus::Unsupported => RequiredDestinationStatus::Unsupported, + CaseStatus::Skipped => RequiredDestinationStatus::Skipped, + _ if !report.missing.is_empty() || !report.missing_unproven.is_empty() => { + RequiredDestinationStatus::Missing + } + _ => RequiredDestinationStatus::Found, + } +} + +fn best_navigation_destination_status<'a>( + reports: impl IntoIterator, +) -> RequiredDestinationStatus { + let statuses = reports + .into_iter() + .map(navigation_destination_status) + .collect::>(); + if statuses.contains(&RequiredDestinationStatus::Found) { + RequiredDestinationStatus::Found + } else if statuses.contains(&RequiredDestinationStatus::Missing) { + RequiredDestinationStatus::Missing + } else if statuses.contains(&RequiredDestinationStatus::Error) { + RequiredDestinationStatus::Error + } else if statuses.contains(&RequiredDestinationStatus::Skipped) { + RequiredDestinationStatus::Skipped + } else { + RequiredDestinationStatus::Unsupported + } +} + +fn navigation_destination_status(report: &UsageDefinitionReport) -> RequiredDestinationStatus { + match report.status { + CaseStatus::Error => RequiredDestinationStatus::Error, + CaseStatus::Unsupported => RequiredDestinationStatus::Unsupported, + CaseStatus::Skipped => RequiredDestinationStatus::Skipped, + CaseStatus::Passed | CaseStatus::PositionUnverified => RequiredDestinationStatus::Found, + _ if best_location_match(&report.actual_declarations, &report.expected_declaration) + != LocationMatch::None => + { + RequiredDestinationStatus::Found + } + _ => RequiredDestinationStatus::Missing, + } +} + +fn type_destination_status(report: &TypeLookupReport) -> RequiredDestinationStatus { + match report.status { + CaseStatus::Error => RequiredDestinationStatus::Error, + CaseStatus::Unsupported => RequiredDestinationStatus::Unsupported, + CaseStatus::Skipped => RequiredDestinationStatus::Skipped, + CaseStatus::Passed | CaseStatus::PositionUnverified => RequiredDestinationStatus::Found, + _ if best_location_match(&report.actual_types, &report.expected_type) + != LocationMatch::None => + { + RequiredDestinationStatus::Found + } + _ => RequiredDestinationStatus::Missing, + } +} + +fn combine_required_destination_statuses( + statuses: impl IntoIterator, +) -> RequiredDestinationStatus { + let statuses = statuses.into_iter().collect::>(); + if statuses.contains(&RequiredDestinationStatus::Error) { + RequiredDestinationStatus::Error + } else if statuses.contains(&RequiredDestinationStatus::Skipped) { + RequiredDestinationStatus::Skipped + } else if statuses.contains(&RequiredDestinationStatus::Unsupported) { + RequiredDestinationStatus::Unsupported + } else if statuses.contains(&RequiredDestinationStatus::Missing) { + RequiredDestinationStatus::Missing + } else { + RequiredDestinationStatus::Found + } +} + +fn combine_case_status_from_statuses( + navigation_statuses: impl IntoIterator, + type_statuses: impl IntoIterator, +) -> CaseStatus { + let statuses = navigation_statuses .into_iter() - .map(|report| report.status) - .chain(definitions.iter().map(|report| report.status)) - .chain(types.iter().map(|report| report.status)) + .chain(type_statuses) .collect::>(); if statuses.contains(&CaseStatus::Error) { CaseStatus::Error @@ -783,6 +947,27 @@ pub(crate) fn compute_totals(documents: &[DocumentRunReport]) -> RunTotals { CaseStatus::Skipped => totals.skipped += 1, CaseStatus::Error => totals.errors += 1, } + match case.required_destination_status { + Some(RequiredDestinationStatus::Found) => { + totals.required_destinations.scoreable_cases += 1; + totals.required_destinations.found += 1; + } + Some(RequiredDestinationStatus::Missing) => { + totals.required_destinations.scoreable_cases += 1; + totals.required_destinations.missing += 1; + } + Some(RequiredDestinationStatus::NotPlanned) => { + totals.required_destinations.not_planned += 1 + } + Some(RequiredDestinationStatus::Unsupported) => { + totals.required_destinations.unsupported += 1 + } + Some(RequiredDestinationStatus::Skipped) => { + totals.required_destinations.skipped += 1 + } + Some(RequiredDestinationStatus::Error) => totals.required_destinations.errors += 1, + None => totals.required_destinations.unreported += 1, + } } } totals diff --git a/src/runners/report_compare.rs b/src/runners/report_compare.rs index 33f39e3..ee97072 100644 --- a/src/runners/report_compare.rs +++ b/src/runners/report_compare.rs @@ -432,6 +432,10 @@ mod tests { unsupported_reason: None, declaration_to_usages: None, usage_to_declaration: Vec::new(), + compatible_usage_to_declaration: Vec::new(), + required_destination_status: Some( + super::super::RequiredDestinationStatus::Found, + ), type_lookups: Vec::new(), diagnostics: Vec::new(), }],