From 431ee9cd7d6591dc3fb1e28f5f2275515e4adfea Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 29 May 2026 16:28:21 +0100 Subject: [PATCH 1/2] Support creating new bugs for web-features Initially create new bugs only for web-features which are: * Supported in both Chrome and Safari * Not supported in Firefox In practice there shouldn't be any of these, although there are a couple of false positives (features that are incorrectly marked as unsupported in Firefox) that make it a useful test case. The plan is to extend this to file bugs more cases in the future. --- bugbot/rules/web_platform_features.py | 254 ++++++++++++++++++++++++-- templates/web_platform_features.html | 208 +++++++++++++-------- 2 files changed, 370 insertions(+), 92 deletions(-) diff --git a/bugbot/rules/web_platform_features.py b/bugbot/rules/web_platform_features.py index 36de880c2..6f91741a4 100644 --- a/bugbot/rules/web_platform_features.py +++ b/bugbot/rules/web_platform_features.py @@ -13,17 +13,21 @@ Any, Generic, Iterable, + Literal, Mapping, MutableMapping, Optional, Sequence, TypeVar, + cast, ) from urllib import parse from google.cloud import bigquery +from libmozdata.bugzilla import Bugzilla +from requests.exceptions import HTTPError -from bugbot import gcp +from bugbot import gcp, logger, spec_mapping, utils from bugbot.bzcleaner import Bug, BzCleaner Json = None | str | int | float | Sequence["Json"] | Mapping[str, "Json"] @@ -87,6 +91,43 @@ def __bool__(self) -> bool: return bool(self.add or self.remove) +@dataclass +class BugzillaNewBug: + """Representation of a new bug to be created""" + + summary: str + product: str + component: str + description: str + type: Literal["defect"] | Literal["enhancement"] | Literal["task"] + version: str = "unspecified" + keywords: Optional[list[str]] = None + whiteboard: Optional[str] = None + see_also: Optional[list[str]] = None + user_story: Optional[str] = None + url: Optional[str] = None + + def to_json(self) -> Mapping[str, Json]: + rv: dict[str, Json] = { + "summary": self.summary, + "product": self.product, + "component": self.component, + "description": self.description, + "version": self.version, + "type": self.type, + } + for value, name in [ + (self.whiteboard, "status_whiteboard"), + (self.keywords, "keywords"), + (self.see_also, "see_also"), + (self.user_story, "cf_user_story"), + (self.url, "url"), + ]: + if value is not None: + rv[name] = value + return rv + + @dataclass class BugzillaUpdate: """Representation of bug changes for use with the Bugzilla ReST API""" @@ -287,11 +328,36 @@ class FeatureData: supported_browsers: set[str] sp_issue: Optional[int] spec_url: set[str] + name: Optional[str] = None + description: Optional[str] = None def is_supported(self) -> bool: return {"firefox", "firefox_android"}.issubset(self.supported_browsers) +def feature_keywords(feature: FeatureData) -> set[str]: + rv = set() + if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): + rv.add("parity-chrome") + if {"safari", "safari_ios"}.issubset(feature.supported_browsers): + rv.add("parity-safari") + return rv + + +def feature_links(feature: FeatureData) -> set[str]: + links = set( + [ + f"https://web-platform-dx.github.io/web-features-explorer/features/{feature.feature}/" + ] + ) + if feature.sp_issue is not None: + links.add( + f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" + ) + links |= feature.spec_url + return links + + @dataclass class FeatureBug: """Bug that represents a web-feature""" @@ -313,10 +379,7 @@ def expected_keywords(self) -> set[str]: rv.add("web-feature") if not self.is_supported(): for feature in self.features.values(): - if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): - rv.add("parity-chrome") - if {"safari", "safari_ios"}.issubset(feature.supported_browsers): - rv.add("parity-safari") + rv |= feature_keywords(feature) return rv def missing_keywords(self) -> set[str]: @@ -324,15 +387,8 @@ def missing_keywords(self) -> set[str]: def expected_links(self) -> set[str]: links = set() - for feature_name, feature in self.features.items(): - links.add( - f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/" - ) - if feature.sp_issue is not None: - links.add( - f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" - ) - links |= feature.spec_url + for feature in self.features.values(): + links |= feature_links(feature) return links def missing_links(self) -> set[str]: @@ -366,6 +422,78 @@ def remove_links(self) -> set[str]: _DataType = TypeVar("_DataType") +class CreateRule(ABC, Generic[_DataType]): + """Rule for creating new bugs based on BigQuery data""" + + def __init__(self, client: bigquery.Client): + self.client = client + + @abstractmethod + def get_data(self) -> _DataType: + ... + + @abstractmethod + def create(self, data: _DataType) -> Mapping[str, BugzillaNewBug]: + ... + + def run(self) -> Mapping[str, BugzillaNewBug]: + data: _DataType = self.get_data() + return self.create(data) + + +class FirefoxOnlyMissing(CreateRule): + def get_data(self) -> list[FeatureData]: + query = """ +SELECT + features.feature, + features.name, + features.description, + (SELECT ARRAY_AGG(browser) FROM UNNEST(features.support)) AS supported_browsers, + features.spec as spec_url, + sp_mozilla.issue as sp_issue +FROM `web_features.features_latest` AS features +LEFT JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs ON + features.feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature")) +LEFT JOIN `standards_positions.mozilla_standards_positions` AS sp_mozilla ON + sp_mozilla.web_feature = features.feature +WHERE + "safari" in UNNEST(features.support.browser) AND + "chrome" IN UNNEST(features.support.browser) AND + "firefox" NOT IN UNNEST(features.support.browser) AND + bugs.number IS NULL +""" + return [ + FeatureData( + feature=row.feature, + spec_url=set(row.spec_url), + supported_browsers=set(row.supported_browsers), + sp_issue=row.sp_issue, + name=row.name, + description=row.description, + ) + for row in self.client.query(query) + ] + + def create(self, data: list[FeatureData]) -> Mapping[str, BugzillaNewBug]: + rv = {} + for feature in data: + product, component = spec_mapping.map_spec_urls(feature.spec_url) + spec_url = feature.spec_url.pop() + rv[feature.feature] = BugzillaNewBug( + summary=f"[meta] Implement {feature.name}", + product=product, + component=component, + description=f"Implement {feature.name}:\n{feature.description}", + type="enhancement", + keywords=["web-feature"] + list(feature_keywords(feature)), + user_story=f"web-feature: {feature.feature}", + url=spec_url, + see_also=[item for item in feature_links(feature) if item != spec_url], + ) + + return rv + + class UpdateRule(ABC, Generic[_DataType]): """Rule for updating bugs based on BigQuery data""" @@ -598,6 +726,8 @@ def update( class WebPlatformFeatures(BzCleaner): def __init__(self) -> None: super().__init__() + self.create_bugs: dict[str, BugzillaNewBug] = {} + self.bugs_created: dict[int, BugzillaNewBug] = {} self.bug_updates: dict[int, FeatureBugUpdate] = defaultdict(FeatureBugUpdate) def description(self) -> str: @@ -610,24 +740,106 @@ def has_default_products(self) -> bool: return False def columns(self) -> list[str]: - return ["id", "summary", "changes", "whiteboard", "user_story"] + return ["id", "summary", "change_type", "changes", "whiteboard", "user_story"] + + def get_bugs( + self, + date: str = "today", + bug_ids: list[int] = [], + chunk_size: Optional[int] = None, + ) -> dict[str, Mapping[str, Any]]: + bugs = super().get_bugs(date, bug_ids, chunk_size) + + if self.create_bugs: + bugs_for_features = self.get_feature_bugs(set(self.create_bugs.keys())) + else: + bugs_for_features = {} + + for i, (feature, bug) in enumerate(self.create_bugs.items()): + if feature in bugs_for_features: + # A bug was already created for this feature + continue + + if self.dryrun or self.test_mode: + response = {"id": i} + logger.info( + f"A bug '{bug.summary}` would be created with:\n{bug.to_json()}", + ) + else: + try: + response = utils.create_bug(cast(dict, bug.to_json())) + except HTTPError: + logger.error( + f"Failed to create bug '{bug.summary}'", + ) + continue + + bug_id = response["id"] + assert isinstance(bug_id, int) + self.bugs_created[bug_id] = bug + bugs[str(bug_id)] = { + "id": bug_id, + "summary": bug.summary, + "url": bug.url, + "see_also": bug.component, + "keywords": bug.keywords, + "whiteboard": bug.whiteboard, + "cf_user_story": bug.user_story, + "status": "NEW", + "resolution": "", + "change_type": "create", + "changes": bug, + "user_story": bug.user_story, + } + + return bugs + + def get_feature_bugs(self, features: set[str]) -> Mapping[str, int]: + """Get the list of existing bugs for specific features""" + data: dict[str, int] = {} + + def handler(bug: Mapping[str, Any], data: dict[str, int]) -> None: + for _, key, value in parse_user_story(bug["cf_user_story"]): + if key == "web-feature": + if value in features: + data[value] = bug["id"] + + Bugzilla( + { + "keywords": "web-feature", + "keywords_type": "allwords", + "f1": "cf_user_story", + "o1": "substring", + "v1": "web-feature", + "f2": "cf_user_story", + "o2": "anywordssubstr", + "v2": ",".join(features), + }, + bugdata=data, + bughandler=handler, + ).wait() + + return data def handle_bug(self, bug: Bug, data: dict[str, Any]) -> Optional[Bug]: bug_id_str = str(bug["id"]) bug_id_int = int(bug["id"]) - if bug_id_int not in self.bug_updates: - return None - bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if bug_id_int in self.bug_updates: + bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if not bugzilla_update: + return None - if bugzilla_update: self.autofix_changes[bug_id_str] = bugzilla_update.to_json() data[bug_id_str] = { + "change_type": "update", "changes": bugzilla_update, "whiteboard": bug["whiteboard"], "user_story": bug["cf_user_story"], } return bug + elif bug_id_int in self.bugs_created: + return bug return None @@ -648,6 +860,10 @@ def get_bz_params(self, date: str) -> dict[str, str | int | list[str] | list[int def get_bug_updates(self) -> None: project = "moz-fx-dev-dschubert-wckb" client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"]) + + for create_rule in [FirefoxOnlyMissing(client)]: + self.create_bugs.update(create_rule.run()) + for update_rule in [ FeatureRenames(client), InvalidFeatures(client), diff --git a/templates/web_platform_features.html b/templates/web_platform_features.html index 1eba74e32..c4ddc259d 100644 --- a/templates/web_platform_features.html +++ b/templates/web_platform_features.html @@ -10,7 +10,7 @@ - {% for i, (bugid, summary, changes, whiteboard, user_story) in enumerate(data) -%} + {% for i, (bugid, summary, change_type, changes, whiteboard, user_story) in enumerate(data) -%} @@ -19,84 +19,146 @@ {{ summary | e }} - + {% elif change_type == "create" %} +

+ Created bug: +

+ {% endif -%} + + + {% endfor -%} + From e49f5a35a15c435620438f661cb87ceaf4839fc0 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 29 May 2026 16:30:51 +0100 Subject: [PATCH 2/2] Add module for best-guess mapping spec URLs to bugzilla product/component This is useful for filing new bugs for incoming feature requests where we might have one or more spec URLs and want to make an initial guess as to where the bug should live. The initial lookup tree was built using Claude to map web-feature bugs with a known spec URL to the relevant component. This means it's likely to be reliable for future features that exist in existing specs but may be wrong for new specs which don't cleanly fit onto existing patterns. Ultimately it might make more sense to store this data somewhere outside of bugbot so that it's more easily kept up to date. --- bugbot/spec_mapping.py | 845 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 845 insertions(+) create mode 100644 bugbot/spec_mapping.py diff --git a/bugbot/spec_mapping.py b/bugbot/spec_mapping.py new file mode 100644 index 000000000..9729357af --- /dev/null +++ b/bugbot/spec_mapping.py @@ -0,0 +1,845 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# mypy: disallow-untyped-defs +"""Map web specification URLs to Bugzilla (product, component) pairs.""" + +import itertools +import re +from abc import ABC, abstractmethod +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from urllib.parse import SplitResult, urlsplit + +ProductComponent = tuple[str, str] + + +@dataclass +class RuleMatch: + """A resolved (product, component)""" + + position: tuple[int, int] + product: str + component: str + + def __gt__(self, other: "RuleMatch") -> bool: + return (self.position[0], -self.position[1]) > ( + other.position[0], + -other.position[1], + ) + + +_order = itertools.count() + + +class Rule(ABC): + level: int = 0 + + def __init__( + self, + default: ProductComponent | None = None, + sub_rules: Sequence["Rule"] = (), + ): + self.default = default + self.sub_rules = tuple(sub_rules) + self.order = next(_order) + + @abstractmethod + def matches(self, parsed: SplitResult) -> bool: + """Whether this rule applies to its URL component""" + ... + + def get(self, parsed: SplitResult) -> RuleMatch | None: + if not self.matches(parsed): + return None + for sub_rule in self.sub_rules: + match = sub_rule.get(parsed) + if match is not None: + return match + if self.default is not None: + return RuleMatch((self.level, self.order), *self.default) + return None + + +class HostRule(Rule): + level = 0 + + def __init__( + self, + default: ProductComponent | None, + sub_rules: Sequence["PathRule"] = (), + ): + super().__init__(default, sub_rules) + + def matches(self, parsed: SplitResult) -> bool: + # The caller is expected to verify that the host matches + return True + + +class PathRule(Rule): + """Matches the URL path. A ``None`` pattern matches any path.""" + + level = 1 + + def __init__( + self, + path: str | None, + default: ProductComponent | None = None, + sub_rules: Sequence["FragmentRule"] = (), + ): + self.path = re.compile(path) if path is not None else None + super().__init__(default, sub_rules) + + def matches(self, parsed: SplitResult) -> bool: + return self.path is None or self.path.search(parsed.path) is not None + + +class FragmentRule(Rule): + """Matches the URL fragment""" + + level = 2 + + def __init__(self, fragment: str, default: ProductComponent | None = None): + self.fragment = re.compile(fragment) + super().__init__(default) + + def matches(self, parsed: SplitResult) -> bool: + return self.fragment.search(parsed.fragment) is not None + + +RULES: dict[str, HostRule] = { + "drafts.csswg.org": HostRule( + default=("Core", "Layout: General"), + sub_rules=[ + PathRule( + r"/css-backgrounds-4/", + sub_rules=[ + FragmentRule(r"background-clip", ("Core", "Web Painting")), + ], + ), + PathRule( + r"/css-images-5/", + sub_rules=[ + FragmentRule( + r"the-object-view-box", + ("Core", "Layout: Images, Video, and HTML Frames"), + ), + ], + ), + PathRule( + r"/css-images-4/", + sub_rules=[ + FragmentRule( + r"element-notation", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/css-inline-3/", + sub_rules=[ + FragmentRule( + r"leading-trim", ("Core", "CSS Parsing and Computation") + ), + FragmentRule(r"baseline-shift-property", ("Core", "Layout")), + ], + ), + PathRule( + r"/css-overflow-4/", + sub_rules=[ + FragmentRule( + r"line-clamp", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/css-fonts-4/", + sub_rules=[ + FragmentRule(r"math-def", ("Core", "MathML")), + FragmentRule( + r"ui-serif-def", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/css-display-4/", + sub_rules=[ + FragmentRule( + r"display-animation", ("Core", "CSS Transitions and Animations") + ), + ], + ), + PathRule( + r"/css-contain-3/", + sub_rules=[ + FragmentRule( + r"content-visibility-animation", + ("Core", "CSS Transitions and Animations"), + ), + ], + ), + PathRule( + r"/css-pseudo-4/", + sub_rules=[ + FragmentRule( + r"highlight-styling", ("Core", "Layout: Text and Fonts") + ), + FragmentRule( + r"selectordef-(spelling|grammar)-error", ("Core", "Layout") + ), + FragmentRule( + r"details-content-pseudo", ("Core", "DOM: Core & HTML") + ), + ], + ), + PathRule(r"/css-values-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-cascade-", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-color-(\d|adjust|hdr|6)", + default=("Core", "CSS Parsing and Computation"), + ), + PathRule( + r"/css-conditional-", default=("Core", "CSS Parsing and Computation") + ), + PathRule( + r"/css-view-transitions-", + default=("Core", "CSS Parsing and Computation"), + ), + PathRule( + r"/css-highlight-api-", default=("Core", "CSS Parsing and Computation") + ), + PathRule( + r"/mediaqueries-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/css-viewport", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-anchor-position-", + default=("Core", "CSS Parsing and Computation"), + ), + PathRule( + r"/css-position-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/css-sizing-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-display-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-contain-", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-variables-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/css-mixins-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-nesting-", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-namespaces-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/selectors-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-pseudo-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-inline-", default=("Core", "Layout: Block and Inline")), + PathRule(r"/css-break-", default=("Core", "Layout: Block and Inline")), + PathRule(r"/css-text-", default=("Core", "Layout: Text and Fonts")), + PathRule(r"/css-fonts-", default=("Core", "Layout: Text and Fonts")), + PathRule(r"/css-text-decor-", default=("Core", "Layout: Text and Fonts")), + PathRule( + r"/css-writing-modes-", default=("Core", "Layout: Text and Fonts") + ), + PathRule(r"/css-size-adjust-", default=("Core", "Layout: Text and Fonts")), + PathRule(r"/css-rhythm-", default=("Core", "Layout: Text and Fonts")), + PathRule( + r"/css-counter-styles", default=("Core", "Layout: Text and Fonts") + ), + PathRule(r"/css-tables-", default=("Core", "Layout: Tables")), + PathRule(r"/css-grid-", default=("Core", "Layout: Grid")), + PathRule(r"/css-flexbox-", default=("Core", "Layout")), + PathRule(r"/css-multicol", default=("Core", "Layout")), + PathRule(r"/css-ruby-", default=("Core", "Layout: Ruby")), + PathRule( + r"/css-scroll-", default=("Core", "Layout: Scrolling and Overflow") + ), + PathRule( + r"/css-overflow-", default=("Core", "Layout: Scrolling and Overflow") + ), + PathRule( + r"/css-overscroll", default=("Core", "Layout: Scrolling and Overflow") + ), + PathRule(r"/scroll-animations", default=("Core", "DOM: Animation")), + PathRule( + r"/css-images-", + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule(r"/css-shapes-", default=("Core", "Layout")), + PathRule(r"/css-masking-", default=("Core", "SVG")), + PathRule(r"/filter-effects-", default=("Core", "SVG")), + PathRule(r"/compositing-", default=("Core", "Web Painting")), + PathRule(r"/geometry-", default=("Core", "DOM: CSS Object Model")), + PathRule(r"/css-page-", default=("Core", "Printing: Output")), + PathRule( + r"/css-animations-", default=("Core", "CSS Transitions and Animations") + ), + PathRule( + r"/css-transitions-", default=("Core", "CSS Transitions and Animations") + ), + PathRule( + r"/css-easing-", default=("Core", "CSS Transitions and Animations") + ), + PathRule( + r"/css-transforms-", default=("Core", "CSS Transitions and Animations") + ), + PathRule(r"/web-animations-", default=("Core", "DOM: Animation")), + PathRule(r"/motion-", default=("Core", "DOM: Animation")), + PathRule(r"/css-will-change", default=("Core", "Web Painting")), + PathRule(r"/css-shadow", default=("Core", "Web Painting")), + PathRule(r"/cssom-view", default=("Core", "Layout")), + PathRule(r"/cssom", default=("Core", "DOM: CSS Object Model")), + PathRule(r"/resize-observer", default=("Core", "Layout")), + PathRule(r"/css-gaps-", default=("Core", "Layout")), + PathRule(r"/css-box-", default=("Core", "Layout")), + PathRule(r"/css-align-", default=("Core", "Layout")), + PathRule(r"/css-logical-", default=("Core", "Layout")), + PathRule(r"/css-env-", default=("Core", "Layout")), + PathRule(r"/css-forms-", default=("Core", "Layout")), + PathRule(r"/css-ui-", default=("Core", "Layout")), + PathRule(r"/css-speech-", default=("Core", "Disability Access APIs")), + PathRule(r"/mathml", default=("Core", "MathML")), + ], + ), + "w3c.github.io": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + r"/webvtt/", + sub_rules=[ + FragmentRule( + r"the-past-and-future-pseudo-classes", ("Core", "Layout") + ), + ], + ), + PathRule(r"/manifest", default=("Firefox", "General")), + PathRule(r"/badging", default=("Firefox for Android", "PWA")), + PathRule(r"/web-share-target", default=("Core", "DOM: Web Share")), + PathRule(r"/web-share", default=("Core", "DOM: Web Share")), + PathRule( + r"/(accelerometer|ambient-light|gyroscope|magnetometer|orientation-sensor|sensors|compute-pressure|deviceorientation|device-posture|battery)", + default=("Core", "DOM: Device Interfaces"), + ), + PathRule(r"/aria", default=("Core", "Disability Access APIs")), + PathRule( + r"/clipboard-apis", + default=("Core", "DOM: Copy & Paste and Drag & Drop"), + ), + PathRule(r"/editing", default=("Core", "DOM: Editor")), + PathRule(r"/edit-context", default=("Core", "DOM: Editor")), + PathRule( + r"/webappsec-permissions-policy", default=("Core", "DOM: Core & HTML") + ), + PathRule( + r"/webappsec-credential-management", + default=("Core", "DOM: Credential Management"), + ), + PathRule(r"/webappsec-referrer-policy", default=("Core", "Networking")), + PathRule(r"/webappsec-", default=("Core", "DOM: Security")), + PathRule(r"/trusted-types", default=("Core", "DOM: Security")), + PathRule(r"/webcrypto", default=("Core", "DOM: Security")), + PathRule( + r"/(hr-time|performance-timeline|user-timing|server-timing|navigation-timing|resource-timing|element-timing|event-timing|paint-timing|largest-contentful-paint|long-animation-frames)", + default=("Core", "DOM: Performance APIs"), + ), + PathRule( + r"/(requestidlecallback|longtasks)", + default=("Core", "DOM: Core & HTML"), + ), + PathRule(r"/IntersectionObserver", default=("Core", "Layout")), + PathRule(r"/IndexedDB", default=("Core", "Storage: IndexedDB")), + PathRule(r"/FileAPI", default=("Core", "DOM: File")), + PathRule(r"/web-locks", default=("Core", "DOM: Core & HTML")), + PathRule( + r"/mediacapture-transform", default=("Core", "WebRTC: Audio/Video") + ), + PathRule(r"/mediacapture-", default=("Core", "WebRTC: Audio/Video")), + PathRule(r"/webrtc", default=("Core", "WebRTC")), + PathRule(r"/webtransport", default=("Core", "Networking")), + PathRule(r"/audio-session", default=("Core", "Audio/Video: Playback")), + PathRule(r"/encrypted-media", default=("Core", "Audio/Video: Playback")), + PathRule( + r"/media-(capabilities|playback-quality|source)", + default=("Core", "Audio/Video: Playback"), + ), + PathRule(r"/mediasession", default=("Core", "Audio/Video: Playback")), + PathRule(r"/webcodecs", default=("Core", "Audio/Video: Playback")), + PathRule(r"/remote-playback", default=("Core", "Audio/Video")), + PathRule(r"/webvtt", default=("Core", "Audio/Video: Playback")), + PathRule( + r"/html-media-capture", default=("Core", "Audio/Video: Recording") + ), + PathRule(r"/picture-in-picture", default=("Core", "DOM: Core & HTML")), + PathRule(r"/mathml", default=("Core", "MathML")), + PathRule(r"/svgwg", default=("Core", "SVG")), + PathRule(r"/ServiceWorker", default=("Core", "DOM: Service Workers")), + PathRule(r"/push-api", default=("Core", "DOM: Push Subscriptions")), + PathRule(r"/gamepad", default=("Core", "DOM: Device Interfaces")), + PathRule(r"/geolocation", default=("Core", "DOM: Geolocation")), + PathRule( + r"/(pointerevents|pointerlock|touch-events|uievents)", + default=("Core", "DOM: UI Events & Focus Handling"), + ), + PathRule(r"/vibration", default=("Core", "DOM: Device Interfaces")), + PathRule( + r"/screen-orientation", default=("Core", "DOM: Device Interfaces") + ), + PathRule(r"/screen-wake-lock", default=("Core", "DOM: Device Interfaces")), + PathRule(r"/virtual-keyboard", default=("Core", "Layout")), + PathRule(r"/window-management", default=("Core", "DOM: Core & HTML")), + PathRule( + r"/(payment-request|web-based-payment-handler|secure-payment-confirmation)", + default=("Core", "DOM: Web Payments"), + ), + PathRule(r"/webauthn", default=("Core", "DOM: Web Authentication")), + PathRule(r"/selection-api", default=("Core", "DOM: Core & HTML")), + PathRule(r"/DOM-Parsing", default=("Core", "DOM: Serializers")), + PathRule(r"/reporting", default=("Core", "DOM: Core & HTML")), + PathRule(r"/presentation-api", default=("Core", "DOM: Core & HTML")), + PathRule(r"/webdriver-bidi", default=("Remote Protocol", "WebDriver BiDi")), + PathRule(r"/webdriver", default=("Remote Protocol", "Marionette")), + PathRule(r"/beacon", default=("Core", "Networking")), + PathRule(r"/permissions", default=("Core", "DOM: Core & HTML")), + PathRule(r"/gpc", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/png", default=("Core", "Graphics: ImageLib")), + ], + ), + "html.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + r"/multipage/webappapis\.html", + sub_rules=[ + FragmentRule( + r"(css|json)-module", ("Core", "DOM: CSS Object Model") + ), + FragmentRule( + r"javascript-module|import-maps", ("Core", "JavaScript Engine") + ), + ], + ), + PathRule( + r"/multipage/semantics-other\.html", + sub_rules=[ + FragmentRule( + r"selector-autofill", ("Core", "CSS Parsing and Computation") + ), + FragmentRule(r"pseudo-classes", ("Core", "Audio/Video: Playback")), + ], + ), + PathRule( + r"/multipage/urls-and-fetching\.html", + sub_rules=[ + FragmentRule(r"blocking", ("Core", "Layout")), + ], + default=("Core", "Networking"), + ), + PathRule( + r"/multipage/links\.html", + sub_rules=[ + FragmentRule(r"ping", ("Core", "DOM: Navigation")), + FragmentRule( + r"link-type-(prefetch|dns-prefetch|preconnect|preload|modulepreload)", + ("Core", "Networking"), + ), + ], + ), + PathRule( + r"/multipage/custom-elements\.html", + sub_rules=[ + FragmentRule( + r"preserving-custom-element-state-when-moved", + ("Core", "DOM: Core & HTML"), + ), + ], + ), + PathRule( + r"/multipage/speculative-loading\.html", + default=("Core", "DOM: Navigation"), + ), + PathRule( + r"/multipage/system-state\.html", + sub_rules=[ + FragmentRule(r"cookies", ("Core", "Networking: Cookies")), + FragmentRule(r"pdf-viewing-support", ("Toolkit", "PDF Viewer")), + ], + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/dom\.html", + sub_rules=[ + FragmentRule(r"attr-translate", ("Core", "DOM: Core & HTML")), + FragmentRule(r"attr-lang", ("Core", "Layout: Text and Fonts")), + FragmentRule( + r"the-style-attribute", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/multipage/iframe-embed-object\.html", + sub_rules=[ + FragmentRule(r"attr-iframe-sandbox", ("Core", "DOM: Security")), + ], + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule( + r"/multipage/form-elements\.html", + sub_rules=[ + FragmentRule(r"attr-button-command", ("Core", "DOM: Core & HTML")), + ], + ), + PathRule(r"/multipage/popover\.html", default=("Core", "DOM: Core & HTML")), + PathRule( + r"/multipage/interaction\.html", + sub_rules=[ + FragmentRule(r"close-requests", ("Core", "DOM: Core & HTML")), + ], + ), + PathRule( + r"/multipage/canvas\.html", default=("Core", "Graphics: Canvas2D") + ), + PathRule( + r"/multipage/workers\.html", default=("Core", "DOM: Service Workers") + ), + PathRule( + r"/multipage/web-messaging\.html", default=("Core", "DOM: postMessage") + ), + PathRule( + r"/multipage/server-sent-events\.html", default=("Core", "Networking") + ), + PathRule( + r"/multipage/nav-history-apis\.html", + default=("Core", "DOM: Navigation"), + ), + PathRule( + r"/multipage/scripting\.html", default=("Core", "DOM: Core & HTML") + ), + PathRule( + r"/multipage/imagebitmap-and-animations\.html", + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/timers-and-user-prompts\.html", + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/structured-data\.html", + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/dynamic-markup-insertion\.html", + default=("Core", "DOM: Serializers"), + ), + PathRule( + r"/multipage/media\.html", default=("Core", "Audio/Video: Playback") + ), + PathRule( + r"/multipage/embedded-content\.html", + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule( + r"/multipage/image-maps\.html", + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule(r"/multipage/tables\.html", default=("Core", "Layout: Tables")), + PathRule( + r"/multipage/dnd\.html", + default=("Core", "DOM: Copy & Paste and Drag & Drop"), + ), + PathRule( + r"/multipage/(forms|form-elements|form-control-infrastructure|input)\.html", + default=("Core", "DOM: Forms"), + ), + ], + ), + "drafts.css-houdini.org": HostRule( + default=("Core", "CSS Parsing and Computation"), + sub_rules=[ + PathRule( + r"/css-paint-api", default=("Core", "CSS Parsing and Computation") + ), + ], + ), + "tc39.es": HostRule( + default=("Core", "JavaScript Engine"), + sub_rules=[ + PathRule( + r"/ecma402", default=("Core", "JavaScript: Internationalization API") + ), + ], + ), + "github.com": HostRule( + default=None, + sub_rules=[ + PathRule(r"/tc39/", default=("Core", "JavaScript Engine")), + PathRule(r"/WebAssembly/", default=("Core", "JavaScript: WebAssembly")), + PathRule(r"/whatwg/(html|dom)", default=("Core", "DOM: Core & HTML")), + PathRule(r"/w3c/manifest", default=("Firefox", "General")), + PathRule(r"/WICG/html-in-canvas", default=("Core", "Graphics: Canvas2D")), + PathRule(r"/WICG/PEPC", default=("Core", "DOM: Core & HTML")), + PathRule(r"/WICG/install-element", default=("Firefox", "General")), + PathRule( + r"/WICG/privacy-preserving-ads", + default=("Core", "Privacy: Anti-Tracking"), + ), + PathRule( + r"/MicrosoftEdge/MSEdgeExplainers/.*OpaqueRange", + default=("Core", "DOM: Forms"), + ), + PathRule( + r"/MicrosoftEdge/MSEdgeExplainers/.*AriaNotify", + default=("Core", "Disability Access APIs"), + ), + PathRule( + r"/MicrosoftEdge/MSEdgeExplainers/.*DocumentSubtitle", + default=("Core", "DOM: Core & HTML"), + ), + ], + ), + "webassembly.github.io": HostRule( + default=("Core", "JavaScript: WebAssembly"), + ), + "registry.khronos.org": HostRule( + default=("Core", "Graphics: CanvasWebGL"), + ), + "gpuweb.github.io": HostRule( + default=("Core", "Graphics: WebGPU"), + ), + "webaudio.github.io": HostRule( + default=("Core", "Web Audio"), + sub_rules=[ + PathRule(r"/web-speech-api", default=("Core", "DOM: Device Interfaces")), + PathRule(r"/web-midi-api", default=("Core", "DOM: Device Interfaces")), + ], + ), + "immersive-web.github.io": HostRule( + default=("Core", "WebVR"), + ), + "dom.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + None, + sub_rules=[ + FragmentRule(r"xpath", ("Core", "XSLT")), + FragmentRule(r"xslt", ("Core", "XSLT")), + ], + ), + ], + ), + "streams.spec.whatwg.org": HostRule( + default=("Core", "DOM: Streams"), + ), + "fetch.spec.whatwg.org": HostRule( + default=("Core", "DOM: Networking"), + sub_rules=[ + PathRule( + None, + sub_rules=[ + FragmentRule(r"dom-window-fetchlater", ("Core", "Networking")), + FragmentRule(r"http-cors", ("Core", "Networking: HTTP")), + ], + ), + ], + ), + "cookiestore.spec.whatwg.org": HostRule( + default=("Core", "Networking: Cookies"), + ), + "url.spec.whatwg.org": HostRule( + default=("Core", "Networking"), + ), + "urlpattern.spec.whatwg.org": HostRule( + default=("Core", "Networking"), + ), + "xhr.spec.whatwg.org": HostRule( + default=("Core", "DOM: Networking"), + ), + "websockets.spec.whatwg.org": HostRule( + default=("Core", "Networking: WebSockets"), + ), + "storage.spec.whatwg.org": HostRule( + default=("Core", "Storage: Quota Manager"), + ), + "fs.spec.whatwg.org": HostRule( + default=("Core", "DOM: File"), + ), + "notifications.spec.whatwg.org": HostRule( + default=("Core", "DOM: Notifications"), + ), + "encoding.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "compression.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "console.spec.whatwg.org": HostRule( + default=("DevTools", "Console"), + ), + "fullscreen.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "webidl.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "compat.spec.whatwg.org": HostRule( + default=("Core", "CSS Parsing and Computation"), + ), + "httpwg.org": HostRule( + default=("Core", "Networking: HTTP"), + sub_rules=[ + PathRule(r"/specs/rfc9842", default=("Core", "Networking: Cache")), + ], + ), + "www.rfc-editor.org": HostRule( + default=("Core", "Networking"), + ), + "w3c-fedid.github.io": HostRule( + default=("Core", "DOM: Credential Management"), + ), + "w3c-cg.github.io": HostRule( + default=None, + sub_rules=[ + PathRule(r"/web-nfc", default=("Core", "DOM: Device Interfaces")), + ], + ), + "webbluetoothcg.github.io": HostRule( + default=("Core", "DOM: Device Interfaces"), + ), + "wicg.github.io": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + r"/attribution-reporting", default=("Core", "Privacy: Anti-Tracking") + ), + PathRule(r"/first-party-sets", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/turtledove", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/shared-storage", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/manifest-incubations", default=("Firefox", "General")), + PathRule(r"/web-app-launch", default=("Firefox", "General")), + PathRule(r"/get-installed-related-apps", default=("Firefox", "General")), + PathRule( + r"/(local-network-access|private-network-access|netinfo|savedata)", + default=("Core", "DOM: Networking"), + ), + PathRule(r"/ua-client-hints", default=("Core", "Networking: HTTP")), + PathRule( + r"/(is-input-pending|performance-measure-memory|js-self-profiling|scheduling-apis)", + default=("Core", "DOM: Performance APIs"), + ), + PathRule(r"/layout-instability", default=("Core", "Layout")), + PathRule(r"/storage-buckets", default=("Core", "Storage: Quota Manager")), + PathRule( + r"/(background-fetch|background-sync|periodic-background-sync|content-index)", + default=("Core", "DOM: Service Workers"), + ), + PathRule(r"/sanitizer-api", default=("Core", "DOM: Security")), + PathRule(r"/signature-based-sri", default=("Core", "DOM: Security")), + PathRule(r"/webcrypto-secure-curves", default=("Core", "DOM: Security")), + PathRule(r"/web-otp", default=("Core", "DOM: Web Authentication")), + PathRule( + r"/(serial|webhid|webusb|shape-detection-api|idle-detection)", + default=("Core", "DOM: Device Interfaces"), + ), + PathRule( + r"/keyboard-(lock|map)", + default=("Core", "DOM: UI Events & Focus Handling"), + ), + PathRule( + r"/ink-enhancement", default=("Core", "DOM: UI Events & Focus Handling") + ), + PathRule(r"/entries-api", default=("Core", "DOM: File")), + PathRule(r"/eyedropper-api", default=("Core", "DOM: Core & HTML")), + PathRule(r"/local-font-access", default=("Core", "Layout: Text and Fonts")), + PathRule( + r"/(crash-reporting|deprecation-reporting|intervention-reporting)", + default=("Core", "DOM: Core & HTML"), + ), + PathRule(r"/nav-speculation", default=("Core", "DOM: Navigation")), + PathRule(r"/scroll-to-text-fragment", default=("Core", "DOM: Navigation")), + PathRule(r"/video-rvfc", default=("Core", "Audio/Video: Playback")), + PathRule( + r"/document-picture-in-picture", default=("Core", "DOM: Core & HTML") + ), + PathRule(r"/controls-list", default=("Core", "Audio/Video: Playback")), + PathRule(r"/file-system-access", default=("WebExtensions", "General")), + PathRule(r"/digital-goods", default=("Core", "DOM: Web Payments")), + PathRule(r"/anonymous-iframe", default=("Core", "DOM: Core & HTML")), + PathRule(r"/fenced-frame", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/portals", default=("Core", "DOM: Core & HTML")), + PathRule(r"/PEPC", default=("Core", "DOM: Core & HTML")), + PathRule(r"/window-controls-overlay", default=("Firefox", "General")), + PathRule(r"/observable", default=("Core", "DOM: Core & HTML")), + PathRule(r"/page-lifecycle", default=("Core", "DOM: Core & HTML")), + PathRule(r"/permissions-request", default=("Core", "DOM: Core & HTML")), + ], + ), + "privacycg.github.io": HostRule( + default=("Core", "Privacy: Anti-Tracking"), + ), + "patcg-individual-drafts.github.io": HostRule( + default=("Core", "Privacy: Anti-Tracking"), + ), + "www.iso.org": HostRule( + default=None, + sub_rules=[ + PathRule(r"/standard/", default=("Core", "Graphics: ImageLib")), + ], + ), + "jpeg.org": HostRule( + default=("Core", "Graphics: ImageLib"), + ), + "aomediacodec.github.io": HostRule( + default=("Core", "Graphics: ImageLib"), + ), + "svgwg.org": HostRule( + default=("Core", "SVG"), + ), + "webmachinelearning.github.io": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "screen-share.github.io": HostRule( + default=("Core", "WebRTC: Audio/Video"), + ), + "open-ui.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "www.w3.org": HostRule( + default=None, + sub_rules=[ + PathRule(r"/TR/device-memory", default=("Core", "DOM: Core & HTML")), + PathRule(r"/TR/.*feature-policy", default=("Core", "DOM: Core & HTML")), + PathRule(r"/TR/webnn", default=("Core", "DOM: Core & HTML")), + PathRule(r"/TR/SVG", default=("Core", "SVG")), + PathRule( + r"/TR/.*selectors-", default=("Core", "CSS Parsing and Computation") + ), + PathRule( + r"/TR/DOM-Level-2-Style", default=("Core", "DOM: CSS Object Model") + ), + PathRule(r"/TR/.*html5", default=("Core", "DOM: Core & HTML")), + PathRule(r"/Graphics/GIF", default=("Core", "Graphics: ImageLib")), + ], + ), +} + + +def map_spec(url: str) -> RuleMatch | None: + """Return the most specific :class:`Match` for ``url``, or ``None``.""" + parsed = urlsplit(url) + host_rule = RULES.get(parsed.hostname or "") + if host_rule is None: + return None + return host_rule.get(parsed) + + +def map_spec_urls(urls: Iterable[str]) -> ProductComponent: + """Pick a plausible (product, component) for a feature with given spec URLs.""" + best_match: RuleMatch | None = None + for url in urls: + rule_match = map_spec(url) + if rule_match is None: + continue + if best_match is None or rule_match > best_match: + best_match = rule_match + if best_match is None: + return "Core", "General" + return best_match.product, best_match.component