Skip to content

Commit 0bc6fab

Browse files
proggeramlugproggeramlug
committed
Require capability snapshots in quality evidence
1 parent 7b07a39 commit 0bc6fab

3 files changed

Lines changed: 200 additions & 0 deletions

File tree

‎tools/quality/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,14 @@ recorded rather than invented. Runtime evidence also reports
158158
the shared acceleration/card prefix observable without conflating it with
159159
SSGI-only baking.
160160

161+
Every accepted native telemetry artifact must also contain the complete
162+
`adapter.renderer_capabilities` and `adapter.device_negotiation` snapshots.
163+
The quality runner validates tier identity, granted features/limits, selected
164+
system paths, active platform profile, chosen request, fallback cause, and
165+
requested device limits before comparing images or timings. Missing or
166+
inconsistent capability evidence fails the case rather than producing an
167+
unqualified performance result.
168+
161169
The one debug-capture API snapshots existing render-graph products:
162170

163171
- `hdr-scene`: RGBA16F scene output, converted for review with the same

‎tools/quality/run.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,42 @@
9090
"adapter",
9191
"renderer_paths",
9292
)
93+
RENDERER_CAPABILITY_TIERS = {"baseline", "modern", "high-end"}
94+
RENDERER_CAPABILITY_PATH_KEYS = {
95+
"materials",
96+
"geometry",
97+
"shadows",
98+
"gi",
99+
"reflections",
100+
"anti_aliasing",
101+
"textures",
102+
"path_tracing",
103+
}
104+
RENDERER_CAPABILITY_FEATURE_KEYS = {
105+
"texture_binding_array",
106+
"non_uniform_indexing",
107+
"indirect_first_instance",
108+
"ray_query",
109+
}
110+
RENDERER_CAPABILITY_LIMIT_KEYS = {
111+
"max_binding_array_elements_per_shader_stage",
112+
"max_binding_array_sampler_elements_per_shader_stage",
113+
"max_texture_array_layers",
114+
"max_sampled_textures_per_shader_stage",
115+
"max_samplers_per_shader_stage",
116+
"max_bind_groups",
117+
"max_color_attachments",
118+
}
119+
DEVICE_NEGOTIATION_LIMIT_KEYS = {
120+
"max_bind_groups",
121+
"max_color_attachments",
122+
"max_sampled_textures_per_shader_stage",
123+
"max_samplers_per_shader_stage",
124+
"max_storage_buffers_per_shader_stage",
125+
"max_uniform_buffer_binding_size",
126+
"max_binding_array_elements_per_shader_stage",
127+
"max_binding_array_sampler_elements_per_shader_stage",
128+
}
93129

94130

95131
class QualityError(RuntimeError):
@@ -119,6 +155,95 @@ def sha256_file(path: Path) -> str:
119155
return digest.hexdigest()
120156

121157

158+
def capability_snapshot_failures(adapter: Mapping[str, Any]) -> list[str]:
159+
"""Validate the capability evidence embedded in every native quality run."""
160+
failures: list[str] = []
161+
capability = adapter.get("renderer_capabilities")
162+
if not isinstance(capability, dict):
163+
return ["adapter did not include renderer_capabilities snapshot"]
164+
165+
for key in ("detected", "selected"):
166+
if capability.get(key) not in RENDERER_CAPABILITY_TIERS:
167+
failures.append(f"renderer_capabilities.{key} is not a known tier")
168+
for key in ("requested", "forced"):
169+
value = capability.get(key)
170+
if value is not None and value not in RENDERER_CAPABILITY_TIERS:
171+
failures.append(f"renderer_capabilities.{key} is not null or a known tier")
172+
if adapter.get("capability_tier") != capability.get("selected"):
173+
failures.append("adapter capability_tier does not match selected renderer tier")
174+
forced = capability.get("forced")
175+
if forced is not None and forced != capability.get("selected"):
176+
failures.append("forced renderer tier does not match selected renderer tier")
177+
if capability.get("diagnostic") is not None and not isinstance(
178+
capability.get("diagnostic"), str
179+
):
180+
failures.append("renderer_capabilities.diagnostic is not null or text")
181+
182+
available = capability.get("available")
183+
if not isinstance(available, dict):
184+
failures.append("renderer_capabilities.available is missing")
185+
else:
186+
features = available.get("features")
187+
if not isinstance(features, dict):
188+
failures.append("renderer_capabilities.available.features is missing")
189+
else:
190+
for key in RENDERER_CAPABILITY_FEATURE_KEYS:
191+
if not isinstance(features.get(key), bool):
192+
failures.append(
193+
f"renderer_capabilities.available.features.{key} is not boolean"
194+
)
195+
limits = available.get("limits")
196+
if not isinstance(limits, dict):
197+
failures.append("renderer_capabilities.available.limits is missing")
198+
else:
199+
for key in RENDERER_CAPABILITY_LIMIT_KEYS:
200+
value = limits.get(key)
201+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
202+
failures.append(
203+
f"renderer_capabilities.available.limits.{key} is invalid"
204+
)
205+
206+
paths = capability.get("paths")
207+
if not isinstance(paths, dict):
208+
failures.append("renderer_capabilities.paths is missing")
209+
else:
210+
for key in RENDERER_CAPABILITY_PATH_KEYS:
211+
value = paths.get(key)
212+
if not isinstance(value, str) or not value:
213+
failures.append(f"renderer_capabilities.paths.{key} is missing")
214+
215+
negotiation = adapter.get("device_negotiation")
216+
if not isinstance(negotiation, dict):
217+
failures.append("adapter did not include device_negotiation snapshot")
218+
return failures
219+
for key in ("preferred_tier", "selected_tier"):
220+
if negotiation.get(key) not in RENDERER_CAPABILITY_TIERS:
221+
failures.append(f"device_negotiation.{key} is not a known tier")
222+
if negotiation.get("selected_tier") != capability.get("selected"):
223+
failures.append("device negotiation tier does not match selected renderer tier")
224+
if negotiation.get("profile") not in {"native-full", "folded-mobile"}:
225+
failures.append("device_negotiation.profile is invalid")
226+
if not isinstance(negotiation.get("selected_request"), str) or not negotiation.get(
227+
"selected_request"
228+
):
229+
failures.append("device_negotiation.selected_request is missing")
230+
if negotiation.get("fallback_cause") is not None and not isinstance(
231+
negotiation.get("fallback_cause"), str
232+
):
233+
failures.append("device_negotiation.fallback_cause is not null or text")
234+
if not isinstance(negotiation.get("required_features"), str):
235+
failures.append("device_negotiation.required_features is missing")
236+
limits = negotiation.get("required_limits")
237+
if not isinstance(limits, dict):
238+
failures.append("device_negotiation.required_limits is missing")
239+
else:
240+
for key in DEVICE_NEGOTIATION_LIMIT_KEYS:
241+
value = limits.get(key)
242+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
243+
failures.append(f"device_negotiation.required_limits.{key} is invalid")
244+
return failures
245+
246+
122247
def repo_path(raw: str, *, must_exist: bool = False) -> Path:
123248
path = (REPO_ROOT / raw).resolve()
124249
try:
@@ -607,6 +732,8 @@ def telemetry_contract_failures(
607732
adapter = telemetry.get("adapter")
608733
if not isinstance(adapter, dict) or adapter.get("availability") != "reported":
609734
failures.append("telemetry did not report the native adapter")
735+
else:
736+
failures.extend(capability_snapshot_failures(adapter))
610737
if not isinstance(telemetry.get("renderer_paths"), dict):
611738
failures.append("telemetry did not report active renderer paths")
612739
for key in ("cpu_frame_mean_ms", "cpu_frame_p95_ms", "measurement_wall_ms"):

‎tools/quality/test_run.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,71 @@ def test_timing_delta_accepts_absolute_or_relative_bound(self) -> None:
247247
self.assertTrue(relative["passed"])
248248
self.assertFalse(rejected["passed"])
249249

250+
def test_capability_snapshot_contract_accepts_complete_native_evidence(self) -> None:
251+
tier = "modern"
252+
adapter = {
253+
"availability": "reported",
254+
"capability_tier": tier,
255+
"renderer_capabilities": {
256+
"detected": tier,
257+
"selected": tier,
258+
"requested": None,
259+
"forced": None,
260+
"diagnostic": None,
261+
"available": {
262+
"features": {
263+
key: False
264+
for key in quality.RENDERER_CAPABILITY_FEATURE_KEYS
265+
},
266+
"limits": {
267+
key: 16
268+
for key in quality.RENDERER_CAPABILITY_LIMIT_KEYS
269+
},
270+
},
271+
"paths": {
272+
key: f"test-{key}"
273+
for key in quality.RENDERER_CAPABILITY_PATH_KEYS
274+
},
275+
},
276+
"device_negotiation": {
277+
"preferred_tier": tier,
278+
"selected_tier": tier,
279+
"profile": "native-full",
280+
"selected_request": "bloom_device_preferred",
281+
"fallback_cause": None,
282+
"required_features": "Features(0x0)",
283+
"required_limits": {
284+
key: 16
285+
for key in quality.DEVICE_NEGOTIATION_LIMIT_KEYS
286+
},
287+
},
288+
}
289+
self.assertEqual(quality.capability_snapshot_failures(adapter), [])
290+
291+
def test_capability_snapshot_contract_rejects_missing_or_inconsistent_data(
292+
self,
293+
) -> None:
294+
self.assertIn(
295+
"adapter did not include renderer_capabilities snapshot",
296+
quality.capability_snapshot_failures({"availability": "reported"}),
297+
)
298+
adapter = {
299+
"capability_tier": "high-end",
300+
"renderer_capabilities": {
301+
"detected": "modern",
302+
"selected": "modern",
303+
"requested": None,
304+
"forced": None,
305+
"diagnostic": None,
306+
"available": {"features": {}, "limits": {}},
307+
"paths": {},
308+
},
309+
"device_negotiation": None,
310+
}
311+
failures = quality.capability_snapshot_failures(adapter)
312+
self.assertTrue(any("does not match selected renderer tier" in item for item in failures))
313+
self.assertTrue(any("device_negotiation snapshot" in item for item in failures))
314+
250315
def test_telemetry_contract_rejects_vsync_and_wrong_frame_count(self) -> None:
251316
case = {
252317
"fixed_timestep": 1 / 60,

0 commit comments

Comments
 (0)