diff --git a/bugbot/rules/web_platform_features.py b/bugbot/rules/web_platform_features.py index 36de880c2..6f91741a4 100644 --- a/bugbot/rules/web_platform_features.py +++ b/bugbot/rules/web_platform_features.py @@ -13,17 +13,21 @@ Any, Generic, Iterable, + Literal, Mapping, MutableMapping, Optional, Sequence, TypeVar, + cast, ) from urllib import parse from google.cloud import bigquery +from libmozdata.bugzilla import Bugzilla +from requests.exceptions import HTTPError -from bugbot import gcp +from bugbot import gcp, logger, spec_mapping, utils from bugbot.bzcleaner import Bug, BzCleaner Json = None | str | int | float | Sequence["Json"] | Mapping[str, "Json"] @@ -87,6 +91,43 @@ def __bool__(self) -> bool: return bool(self.add or self.remove) +@dataclass +class BugzillaNewBug: + """Representation of a new bug to be created""" + + summary: str + product: str + component: str + description: str + type: Literal["defect"] | Literal["enhancement"] | Literal["task"] + version: str = "unspecified" + keywords: Optional[list[str]] = None + whiteboard: Optional[str] = None + see_also: Optional[list[str]] = None + user_story: Optional[str] = None + url: Optional[str] = None + + def to_json(self) -> Mapping[str, Json]: + rv: dict[str, Json] = { + "summary": self.summary, + "product": self.product, + "component": self.component, + "description": self.description, + "version": self.version, + "type": self.type, + } + for value, name in [ + (self.whiteboard, "status_whiteboard"), + (self.keywords, "keywords"), + (self.see_also, "see_also"), + (self.user_story, "cf_user_story"), + (self.url, "url"), + ]: + if value is not None: + rv[name] = value + return rv + + @dataclass class BugzillaUpdate: """Representation of bug changes for use with the Bugzilla ReST API""" @@ -287,11 +328,36 @@ class FeatureData: supported_browsers: set[str] sp_issue: Optional[int] spec_url: set[str] + name: Optional[str] = None + description: Optional[str] = None def is_supported(self) -> bool: return {"firefox", "firefox_android"}.issubset(self.supported_browsers) +def feature_keywords(feature: FeatureData) -> set[str]: + rv = set() + if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): + rv.add("parity-chrome") + if {"safari", "safari_ios"}.issubset(feature.supported_browsers): + rv.add("parity-safari") + return rv + + +def feature_links(feature: FeatureData) -> set[str]: + links = set( + [ + f"https://web-platform-dx.github.io/web-features-explorer/features/{feature.feature}/" + ] + ) + if feature.sp_issue is not None: + links.add( + f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" + ) + links |= feature.spec_url + return links + + @dataclass class FeatureBug: """Bug that represents a web-feature""" @@ -313,10 +379,7 @@ def expected_keywords(self) -> set[str]: rv.add("web-feature") if not self.is_supported(): for feature in self.features.values(): - if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): - rv.add("parity-chrome") - if {"safari", "safari_ios"}.issubset(feature.supported_browsers): - rv.add("parity-safari") + rv |= feature_keywords(feature) return rv def missing_keywords(self) -> set[str]: @@ -324,15 +387,8 @@ def missing_keywords(self) -> set[str]: def expected_links(self) -> set[str]: links = set() - for feature_name, feature in self.features.items(): - links.add( - f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/" - ) - if feature.sp_issue is not None: - links.add( - f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" - ) - links |= feature.spec_url + for feature in self.features.values(): + links |= feature_links(feature) return links def missing_links(self) -> set[str]: @@ -366,6 +422,78 @@ def remove_links(self) -> set[str]: _DataType = TypeVar("_DataType") +class CreateRule(ABC, Generic[_DataType]): + """Rule for creating new bugs based on BigQuery data""" + + def __init__(self, client: bigquery.Client): + self.client = client + + @abstractmethod + def get_data(self) -> _DataType: + ... + + @abstractmethod + def create(self, data: _DataType) -> Mapping[str, BugzillaNewBug]: + ... + + def run(self) -> Mapping[str, BugzillaNewBug]: + data: _DataType = self.get_data() + return self.create(data) + + +class FirefoxOnlyMissing(CreateRule): + def get_data(self) -> list[FeatureData]: + query = """ +SELECT + features.feature, + features.name, + features.description, + (SELECT ARRAY_AGG(browser) FROM UNNEST(features.support)) AS supported_browsers, + features.spec as spec_url, + sp_mozilla.issue as sp_issue +FROM `web_features.features_latest` AS features +LEFT JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs ON + features.feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature")) +LEFT JOIN `standards_positions.mozilla_standards_positions` AS sp_mozilla ON + sp_mozilla.web_feature = features.feature +WHERE + "safari" in UNNEST(features.support.browser) AND + "chrome" IN UNNEST(features.support.browser) AND + "firefox" NOT IN UNNEST(features.support.browser) AND + bugs.number IS NULL +""" + return [ + FeatureData( + feature=row.feature, + spec_url=set(row.spec_url), + supported_browsers=set(row.supported_browsers), + sp_issue=row.sp_issue, + name=row.name, + description=row.description, + ) + for row in self.client.query(query) + ] + + def create(self, data: list[FeatureData]) -> Mapping[str, BugzillaNewBug]: + rv = {} + for feature in data: + product, component = spec_mapping.map_spec_urls(feature.spec_url) + spec_url = feature.spec_url.pop() + rv[feature.feature] = BugzillaNewBug( + summary=f"[meta] Implement {feature.name}", + product=product, + component=component, + description=f"Implement {feature.name}:\n{feature.description}", + type="enhancement", + keywords=["web-feature"] + list(feature_keywords(feature)), + user_story=f"web-feature: {feature.feature}", + url=spec_url, + see_also=[item for item in feature_links(feature) if item != spec_url], + ) + + return rv + + class UpdateRule(ABC, Generic[_DataType]): """Rule for updating bugs based on BigQuery data""" @@ -598,6 +726,8 @@ def update( class WebPlatformFeatures(BzCleaner): def __init__(self) -> None: super().__init__() + self.create_bugs: dict[str, BugzillaNewBug] = {} + self.bugs_created: dict[int, BugzillaNewBug] = {} self.bug_updates: dict[int, FeatureBugUpdate] = defaultdict(FeatureBugUpdate) def description(self) -> str: @@ -610,24 +740,106 @@ def has_default_products(self) -> bool: return False def columns(self) -> list[str]: - return ["id", "summary", "changes", "whiteboard", "user_story"] + return ["id", "summary", "change_type", "changes", "whiteboard", "user_story"] + + def get_bugs( + self, + date: str = "today", + bug_ids: list[int] = [], + chunk_size: Optional[int] = None, + ) -> dict[str, Mapping[str, Any]]: + bugs = super().get_bugs(date, bug_ids, chunk_size) + + if self.create_bugs: + bugs_for_features = self.get_feature_bugs(set(self.create_bugs.keys())) + else: + bugs_for_features = {} + + for i, (feature, bug) in enumerate(self.create_bugs.items()): + if feature in bugs_for_features: + # A bug was already created for this feature + continue + + if self.dryrun or self.test_mode: + response = {"id": i} + logger.info( + f"A bug '{bug.summary}` would be created with:\n{bug.to_json()}", + ) + else: + try: + response = utils.create_bug(cast(dict, bug.to_json())) + except HTTPError: + logger.error( + f"Failed to create bug '{bug.summary}'", + ) + continue + + bug_id = response["id"] + assert isinstance(bug_id, int) + self.bugs_created[bug_id] = bug + bugs[str(bug_id)] = { + "id": bug_id, + "summary": bug.summary, + "url": bug.url, + "see_also": bug.component, + "keywords": bug.keywords, + "whiteboard": bug.whiteboard, + "cf_user_story": bug.user_story, + "status": "NEW", + "resolution": "", + "change_type": "create", + "changes": bug, + "user_story": bug.user_story, + } + + return bugs + + def get_feature_bugs(self, features: set[str]) -> Mapping[str, int]: + """Get the list of existing bugs for specific features""" + data: dict[str, int] = {} + + def handler(bug: Mapping[str, Any], data: dict[str, int]) -> None: + for _, key, value in parse_user_story(bug["cf_user_story"]): + if key == "web-feature": + if value in features: + data[value] = bug["id"] + + Bugzilla( + { + "keywords": "web-feature", + "keywords_type": "allwords", + "f1": "cf_user_story", + "o1": "substring", + "v1": "web-feature", + "f2": "cf_user_story", + "o2": "anywordssubstr", + "v2": ",".join(features), + }, + bugdata=data, + bughandler=handler, + ).wait() + + return data def handle_bug(self, bug: Bug, data: dict[str, Any]) -> Optional[Bug]: bug_id_str = str(bug["id"]) bug_id_int = int(bug["id"]) - if bug_id_int not in self.bug_updates: - return None - bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if bug_id_int in self.bug_updates: + bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if not bugzilla_update: + return None - if bugzilla_update: self.autofix_changes[bug_id_str] = bugzilla_update.to_json() data[bug_id_str] = { + "change_type": "update", "changes": bugzilla_update, "whiteboard": bug["whiteboard"], "user_story": bug["cf_user_story"], } return bug + elif bug_id_int in self.bugs_created: + return bug return None @@ -648,6 +860,10 @@ def get_bz_params(self, date: str) -> dict[str, str | int | list[str] | list[int def get_bug_updates(self) -> None: project = "moz-fx-dev-dschubert-wckb" client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"]) + + for create_rule in [FirefoxOnlyMissing(client)]: + self.create_bugs.update(create_rule.run()) + for update_rule in [ FeatureRenames(client), InvalidFeatures(client), diff --git a/bugbot/spec_mapping.py b/bugbot/spec_mapping.py new file mode 100644 index 000000000..9729357af --- /dev/null +++ b/bugbot/spec_mapping.py @@ -0,0 +1,845 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# mypy: disallow-untyped-defs +"""Map web specification URLs to Bugzilla (product, component) pairs.""" + +import itertools +import re +from abc import ABC, abstractmethod +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from urllib.parse import SplitResult, urlsplit + +ProductComponent = tuple[str, str] + + +@dataclass +class RuleMatch: + """A resolved (product, component)""" + + position: tuple[int, int] + product: str + component: str + + def __gt__(self, other: "RuleMatch") -> bool: + return (self.position[0], -self.position[1]) > ( + other.position[0], + -other.position[1], + ) + + +_order = itertools.count() + + +class Rule(ABC): + level: int = 0 + + def __init__( + self, + default: ProductComponent | None = None, + sub_rules: Sequence["Rule"] = (), + ): + self.default = default + self.sub_rules = tuple(sub_rules) + self.order = next(_order) + + @abstractmethod + def matches(self, parsed: SplitResult) -> bool: + """Whether this rule applies to its URL component""" + ... + + def get(self, parsed: SplitResult) -> RuleMatch | None: + if not self.matches(parsed): + return None + for sub_rule in self.sub_rules: + match = sub_rule.get(parsed) + if match is not None: + return match + if self.default is not None: + return RuleMatch((self.level, self.order), *self.default) + return None + + +class HostRule(Rule): + level = 0 + + def __init__( + self, + default: ProductComponent | None, + sub_rules: Sequence["PathRule"] = (), + ): + super().__init__(default, sub_rules) + + def matches(self, parsed: SplitResult) -> bool: + # The caller is expected to verify that the host matches + return True + + +class PathRule(Rule): + """Matches the URL path. A ``None`` pattern matches any path.""" + + level = 1 + + def __init__( + self, + path: str | None, + default: ProductComponent | None = None, + sub_rules: Sequence["FragmentRule"] = (), + ): + self.path = re.compile(path) if path is not None else None + super().__init__(default, sub_rules) + + def matches(self, parsed: SplitResult) -> bool: + return self.path is None or self.path.search(parsed.path) is not None + + +class FragmentRule(Rule): + """Matches the URL fragment""" + + level = 2 + + def __init__(self, fragment: str, default: ProductComponent | None = None): + self.fragment = re.compile(fragment) + super().__init__(default) + + def matches(self, parsed: SplitResult) -> bool: + return self.fragment.search(parsed.fragment) is not None + + +RULES: dict[str, HostRule] = { + "drafts.csswg.org": HostRule( + default=("Core", "Layout: General"), + sub_rules=[ + PathRule( + r"/css-backgrounds-4/", + sub_rules=[ + FragmentRule(r"background-clip", ("Core", "Web Painting")), + ], + ), + PathRule( + r"/css-images-5/", + sub_rules=[ + FragmentRule( + r"the-object-view-box", + ("Core", "Layout: Images, Video, and HTML Frames"), + ), + ], + ), + PathRule( + r"/css-images-4/", + sub_rules=[ + FragmentRule( + r"element-notation", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/css-inline-3/", + sub_rules=[ + FragmentRule( + r"leading-trim", ("Core", "CSS Parsing and Computation") + ), + FragmentRule(r"baseline-shift-property", ("Core", "Layout")), + ], + ), + PathRule( + r"/css-overflow-4/", + sub_rules=[ + FragmentRule( + r"line-clamp", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/css-fonts-4/", + sub_rules=[ + FragmentRule(r"math-def", ("Core", "MathML")), + FragmentRule( + r"ui-serif-def", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/css-display-4/", + sub_rules=[ + FragmentRule( + r"display-animation", ("Core", "CSS Transitions and Animations") + ), + ], + ), + PathRule( + r"/css-contain-3/", + sub_rules=[ + FragmentRule( + r"content-visibility-animation", + ("Core", "CSS Transitions and Animations"), + ), + ], + ), + PathRule( + r"/css-pseudo-4/", + sub_rules=[ + FragmentRule( + r"highlight-styling", ("Core", "Layout: Text and Fonts") + ), + FragmentRule( + r"selectordef-(spelling|grammar)-error", ("Core", "Layout") + ), + FragmentRule( + r"details-content-pseudo", ("Core", "DOM: Core & HTML") + ), + ], + ), + PathRule(r"/css-values-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-cascade-", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-color-(\d|adjust|hdr|6)", + default=("Core", "CSS Parsing and Computation"), + ), + PathRule( + r"/css-conditional-", default=("Core", "CSS Parsing and Computation") + ), + PathRule( + r"/css-view-transitions-", + default=("Core", "CSS Parsing and Computation"), + ), + PathRule( + r"/css-highlight-api-", default=("Core", "CSS Parsing and Computation") + ), + PathRule( + r"/mediaqueries-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/css-viewport", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-anchor-position-", + default=("Core", "CSS Parsing and Computation"), + ), + PathRule( + r"/css-position-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/css-sizing-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-display-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-contain-", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-variables-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/css-mixins-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-nesting-", default=("Core", "CSS Parsing and Computation")), + PathRule( + r"/css-namespaces-", default=("Core", "CSS Parsing and Computation") + ), + PathRule(r"/selectors-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-pseudo-", default=("Core", "CSS Parsing and Computation")), + PathRule(r"/css-inline-", default=("Core", "Layout: Block and Inline")), + PathRule(r"/css-break-", default=("Core", "Layout: Block and Inline")), + PathRule(r"/css-text-", default=("Core", "Layout: Text and Fonts")), + PathRule(r"/css-fonts-", default=("Core", "Layout: Text and Fonts")), + PathRule(r"/css-text-decor-", default=("Core", "Layout: Text and Fonts")), + PathRule( + r"/css-writing-modes-", default=("Core", "Layout: Text and Fonts") + ), + PathRule(r"/css-size-adjust-", default=("Core", "Layout: Text and Fonts")), + PathRule(r"/css-rhythm-", default=("Core", "Layout: Text and Fonts")), + PathRule( + r"/css-counter-styles", default=("Core", "Layout: Text and Fonts") + ), + PathRule(r"/css-tables-", default=("Core", "Layout: Tables")), + PathRule(r"/css-grid-", default=("Core", "Layout: Grid")), + PathRule(r"/css-flexbox-", default=("Core", "Layout")), + PathRule(r"/css-multicol", default=("Core", "Layout")), + PathRule(r"/css-ruby-", default=("Core", "Layout: Ruby")), + PathRule( + r"/css-scroll-", default=("Core", "Layout: Scrolling and Overflow") + ), + PathRule( + r"/css-overflow-", default=("Core", "Layout: Scrolling and Overflow") + ), + PathRule( + r"/css-overscroll", default=("Core", "Layout: Scrolling and Overflow") + ), + PathRule(r"/scroll-animations", default=("Core", "DOM: Animation")), + PathRule( + r"/css-images-", + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule(r"/css-shapes-", default=("Core", "Layout")), + PathRule(r"/css-masking-", default=("Core", "SVG")), + PathRule(r"/filter-effects-", default=("Core", "SVG")), + PathRule(r"/compositing-", default=("Core", "Web Painting")), + PathRule(r"/geometry-", default=("Core", "DOM: CSS Object Model")), + PathRule(r"/css-page-", default=("Core", "Printing: Output")), + PathRule( + r"/css-animations-", default=("Core", "CSS Transitions and Animations") + ), + PathRule( + r"/css-transitions-", default=("Core", "CSS Transitions and Animations") + ), + PathRule( + r"/css-easing-", default=("Core", "CSS Transitions and Animations") + ), + PathRule( + r"/css-transforms-", default=("Core", "CSS Transitions and Animations") + ), + PathRule(r"/web-animations-", default=("Core", "DOM: Animation")), + PathRule(r"/motion-", default=("Core", "DOM: Animation")), + PathRule(r"/css-will-change", default=("Core", "Web Painting")), + PathRule(r"/css-shadow", default=("Core", "Web Painting")), + PathRule(r"/cssom-view", default=("Core", "Layout")), + PathRule(r"/cssom", default=("Core", "DOM: CSS Object Model")), + PathRule(r"/resize-observer", default=("Core", "Layout")), + PathRule(r"/css-gaps-", default=("Core", "Layout")), + PathRule(r"/css-box-", default=("Core", "Layout")), + PathRule(r"/css-align-", default=("Core", "Layout")), + PathRule(r"/css-logical-", default=("Core", "Layout")), + PathRule(r"/css-env-", default=("Core", "Layout")), + PathRule(r"/css-forms-", default=("Core", "Layout")), + PathRule(r"/css-ui-", default=("Core", "Layout")), + PathRule(r"/css-speech-", default=("Core", "Disability Access APIs")), + PathRule(r"/mathml", default=("Core", "MathML")), + ], + ), + "w3c.github.io": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + r"/webvtt/", + sub_rules=[ + FragmentRule( + r"the-past-and-future-pseudo-classes", ("Core", "Layout") + ), + ], + ), + PathRule(r"/manifest", default=("Firefox", "General")), + PathRule(r"/badging", default=("Firefox for Android", "PWA")), + PathRule(r"/web-share-target", default=("Core", "DOM: Web Share")), + PathRule(r"/web-share", default=("Core", "DOM: Web Share")), + PathRule( + r"/(accelerometer|ambient-light|gyroscope|magnetometer|orientation-sensor|sensors|compute-pressure|deviceorientation|device-posture|battery)", + default=("Core", "DOM: Device Interfaces"), + ), + PathRule(r"/aria", default=("Core", "Disability Access APIs")), + PathRule( + r"/clipboard-apis", + default=("Core", "DOM: Copy & Paste and Drag & Drop"), + ), + PathRule(r"/editing", default=("Core", "DOM: Editor")), + PathRule(r"/edit-context", default=("Core", "DOM: Editor")), + PathRule( + r"/webappsec-permissions-policy", default=("Core", "DOM: Core & HTML") + ), + PathRule( + r"/webappsec-credential-management", + default=("Core", "DOM: Credential Management"), + ), + PathRule(r"/webappsec-referrer-policy", default=("Core", "Networking")), + PathRule(r"/webappsec-", default=("Core", "DOM: Security")), + PathRule(r"/trusted-types", default=("Core", "DOM: Security")), + PathRule(r"/webcrypto", default=("Core", "DOM: Security")), + PathRule( + r"/(hr-time|performance-timeline|user-timing|server-timing|navigation-timing|resource-timing|element-timing|event-timing|paint-timing|largest-contentful-paint|long-animation-frames)", + default=("Core", "DOM: Performance APIs"), + ), + PathRule( + r"/(requestidlecallback|longtasks)", + default=("Core", "DOM: Core & HTML"), + ), + PathRule(r"/IntersectionObserver", default=("Core", "Layout")), + PathRule(r"/IndexedDB", default=("Core", "Storage: IndexedDB")), + PathRule(r"/FileAPI", default=("Core", "DOM: File")), + PathRule(r"/web-locks", default=("Core", "DOM: Core & HTML")), + PathRule( + r"/mediacapture-transform", default=("Core", "WebRTC: Audio/Video") + ), + PathRule(r"/mediacapture-", default=("Core", "WebRTC: Audio/Video")), + PathRule(r"/webrtc", default=("Core", "WebRTC")), + PathRule(r"/webtransport", default=("Core", "Networking")), + PathRule(r"/audio-session", default=("Core", "Audio/Video: Playback")), + PathRule(r"/encrypted-media", default=("Core", "Audio/Video: Playback")), + PathRule( + r"/media-(capabilities|playback-quality|source)", + default=("Core", "Audio/Video: Playback"), + ), + PathRule(r"/mediasession", default=("Core", "Audio/Video: Playback")), + PathRule(r"/webcodecs", default=("Core", "Audio/Video: Playback")), + PathRule(r"/remote-playback", default=("Core", "Audio/Video")), + PathRule(r"/webvtt", default=("Core", "Audio/Video: Playback")), + PathRule( + r"/html-media-capture", default=("Core", "Audio/Video: Recording") + ), + PathRule(r"/picture-in-picture", default=("Core", "DOM: Core & HTML")), + PathRule(r"/mathml", default=("Core", "MathML")), + PathRule(r"/svgwg", default=("Core", "SVG")), + PathRule(r"/ServiceWorker", default=("Core", "DOM: Service Workers")), + PathRule(r"/push-api", default=("Core", "DOM: Push Subscriptions")), + PathRule(r"/gamepad", default=("Core", "DOM: Device Interfaces")), + PathRule(r"/geolocation", default=("Core", "DOM: Geolocation")), + PathRule( + r"/(pointerevents|pointerlock|touch-events|uievents)", + default=("Core", "DOM: UI Events & Focus Handling"), + ), + PathRule(r"/vibration", default=("Core", "DOM: Device Interfaces")), + PathRule( + r"/screen-orientation", default=("Core", "DOM: Device Interfaces") + ), + PathRule(r"/screen-wake-lock", default=("Core", "DOM: Device Interfaces")), + PathRule(r"/virtual-keyboard", default=("Core", "Layout")), + PathRule(r"/window-management", default=("Core", "DOM: Core & HTML")), + PathRule( + r"/(payment-request|web-based-payment-handler|secure-payment-confirmation)", + default=("Core", "DOM: Web Payments"), + ), + PathRule(r"/webauthn", default=("Core", "DOM: Web Authentication")), + PathRule(r"/selection-api", default=("Core", "DOM: Core & HTML")), + PathRule(r"/DOM-Parsing", default=("Core", "DOM: Serializers")), + PathRule(r"/reporting", default=("Core", "DOM: Core & HTML")), + PathRule(r"/presentation-api", default=("Core", "DOM: Core & HTML")), + PathRule(r"/webdriver-bidi", default=("Remote Protocol", "WebDriver BiDi")), + PathRule(r"/webdriver", default=("Remote Protocol", "Marionette")), + PathRule(r"/beacon", default=("Core", "Networking")), + PathRule(r"/permissions", default=("Core", "DOM: Core & HTML")), + PathRule(r"/gpc", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/png", default=("Core", "Graphics: ImageLib")), + ], + ), + "html.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + r"/multipage/webappapis\.html", + sub_rules=[ + FragmentRule( + r"(css|json)-module", ("Core", "DOM: CSS Object Model") + ), + FragmentRule( + r"javascript-module|import-maps", ("Core", "JavaScript Engine") + ), + ], + ), + PathRule( + r"/multipage/semantics-other\.html", + sub_rules=[ + FragmentRule( + r"selector-autofill", ("Core", "CSS Parsing and Computation") + ), + FragmentRule(r"pseudo-classes", ("Core", "Audio/Video: Playback")), + ], + ), + PathRule( + r"/multipage/urls-and-fetching\.html", + sub_rules=[ + FragmentRule(r"blocking", ("Core", "Layout")), + ], + default=("Core", "Networking"), + ), + PathRule( + r"/multipage/links\.html", + sub_rules=[ + FragmentRule(r"ping", ("Core", "DOM: Navigation")), + FragmentRule( + r"link-type-(prefetch|dns-prefetch|preconnect|preload|modulepreload)", + ("Core", "Networking"), + ), + ], + ), + PathRule( + r"/multipage/custom-elements\.html", + sub_rules=[ + FragmentRule( + r"preserving-custom-element-state-when-moved", + ("Core", "DOM: Core & HTML"), + ), + ], + ), + PathRule( + r"/multipage/speculative-loading\.html", + default=("Core", "DOM: Navigation"), + ), + PathRule( + r"/multipage/system-state\.html", + sub_rules=[ + FragmentRule(r"cookies", ("Core", "Networking: Cookies")), + FragmentRule(r"pdf-viewing-support", ("Toolkit", "PDF Viewer")), + ], + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/dom\.html", + sub_rules=[ + FragmentRule(r"attr-translate", ("Core", "DOM: Core & HTML")), + FragmentRule(r"attr-lang", ("Core", "Layout: Text and Fonts")), + FragmentRule( + r"the-style-attribute", ("Core", "CSS Parsing and Computation") + ), + ], + ), + PathRule( + r"/multipage/iframe-embed-object\.html", + sub_rules=[ + FragmentRule(r"attr-iframe-sandbox", ("Core", "DOM: Security")), + ], + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule( + r"/multipage/form-elements\.html", + sub_rules=[ + FragmentRule(r"attr-button-command", ("Core", "DOM: Core & HTML")), + ], + ), + PathRule(r"/multipage/popover\.html", default=("Core", "DOM: Core & HTML")), + PathRule( + r"/multipage/interaction\.html", + sub_rules=[ + FragmentRule(r"close-requests", ("Core", "DOM: Core & HTML")), + ], + ), + PathRule( + r"/multipage/canvas\.html", default=("Core", "Graphics: Canvas2D") + ), + PathRule( + r"/multipage/workers\.html", default=("Core", "DOM: Service Workers") + ), + PathRule( + r"/multipage/web-messaging\.html", default=("Core", "DOM: postMessage") + ), + PathRule( + r"/multipage/server-sent-events\.html", default=("Core", "Networking") + ), + PathRule( + r"/multipage/nav-history-apis\.html", + default=("Core", "DOM: Navigation"), + ), + PathRule( + r"/multipage/scripting\.html", default=("Core", "DOM: Core & HTML") + ), + PathRule( + r"/multipage/imagebitmap-and-animations\.html", + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/timers-and-user-prompts\.html", + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/structured-data\.html", + default=("Core", "DOM: Core & HTML"), + ), + PathRule( + r"/multipage/dynamic-markup-insertion\.html", + default=("Core", "DOM: Serializers"), + ), + PathRule( + r"/multipage/media\.html", default=("Core", "Audio/Video: Playback") + ), + PathRule( + r"/multipage/embedded-content\.html", + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule( + r"/multipage/image-maps\.html", + default=("Core", "Layout: Images, Video, and HTML Frames"), + ), + PathRule(r"/multipage/tables\.html", default=("Core", "Layout: Tables")), + PathRule( + r"/multipage/dnd\.html", + default=("Core", "DOM: Copy & Paste and Drag & Drop"), + ), + PathRule( + r"/multipage/(forms|form-elements|form-control-infrastructure|input)\.html", + default=("Core", "DOM: Forms"), + ), + ], + ), + "drafts.css-houdini.org": HostRule( + default=("Core", "CSS Parsing and Computation"), + sub_rules=[ + PathRule( + r"/css-paint-api", default=("Core", "CSS Parsing and Computation") + ), + ], + ), + "tc39.es": HostRule( + default=("Core", "JavaScript Engine"), + sub_rules=[ + PathRule( + r"/ecma402", default=("Core", "JavaScript: Internationalization API") + ), + ], + ), + "github.com": HostRule( + default=None, + sub_rules=[ + PathRule(r"/tc39/", default=("Core", "JavaScript Engine")), + PathRule(r"/WebAssembly/", default=("Core", "JavaScript: WebAssembly")), + PathRule(r"/whatwg/(html|dom)", default=("Core", "DOM: Core & HTML")), + PathRule(r"/w3c/manifest", default=("Firefox", "General")), + PathRule(r"/WICG/html-in-canvas", default=("Core", "Graphics: Canvas2D")), + PathRule(r"/WICG/PEPC", default=("Core", "DOM: Core & HTML")), + PathRule(r"/WICG/install-element", default=("Firefox", "General")), + PathRule( + r"/WICG/privacy-preserving-ads", + default=("Core", "Privacy: Anti-Tracking"), + ), + PathRule( + r"/MicrosoftEdge/MSEdgeExplainers/.*OpaqueRange", + default=("Core", "DOM: Forms"), + ), + PathRule( + r"/MicrosoftEdge/MSEdgeExplainers/.*AriaNotify", + default=("Core", "Disability Access APIs"), + ), + PathRule( + r"/MicrosoftEdge/MSEdgeExplainers/.*DocumentSubtitle", + default=("Core", "DOM: Core & HTML"), + ), + ], + ), + "webassembly.github.io": HostRule( + default=("Core", "JavaScript: WebAssembly"), + ), + "registry.khronos.org": HostRule( + default=("Core", "Graphics: CanvasWebGL"), + ), + "gpuweb.github.io": HostRule( + default=("Core", "Graphics: WebGPU"), + ), + "webaudio.github.io": HostRule( + default=("Core", "Web Audio"), + sub_rules=[ + PathRule(r"/web-speech-api", default=("Core", "DOM: Device Interfaces")), + PathRule(r"/web-midi-api", default=("Core", "DOM: Device Interfaces")), + ], + ), + "immersive-web.github.io": HostRule( + default=("Core", "WebVR"), + ), + "dom.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + None, + sub_rules=[ + FragmentRule(r"xpath", ("Core", "XSLT")), + FragmentRule(r"xslt", ("Core", "XSLT")), + ], + ), + ], + ), + "streams.spec.whatwg.org": HostRule( + default=("Core", "DOM: Streams"), + ), + "fetch.spec.whatwg.org": HostRule( + default=("Core", "DOM: Networking"), + sub_rules=[ + PathRule( + None, + sub_rules=[ + FragmentRule(r"dom-window-fetchlater", ("Core", "Networking")), + FragmentRule(r"http-cors", ("Core", "Networking: HTTP")), + ], + ), + ], + ), + "cookiestore.spec.whatwg.org": HostRule( + default=("Core", "Networking: Cookies"), + ), + "url.spec.whatwg.org": HostRule( + default=("Core", "Networking"), + ), + "urlpattern.spec.whatwg.org": HostRule( + default=("Core", "Networking"), + ), + "xhr.spec.whatwg.org": HostRule( + default=("Core", "DOM: Networking"), + ), + "websockets.spec.whatwg.org": HostRule( + default=("Core", "Networking: WebSockets"), + ), + "storage.spec.whatwg.org": HostRule( + default=("Core", "Storage: Quota Manager"), + ), + "fs.spec.whatwg.org": HostRule( + default=("Core", "DOM: File"), + ), + "notifications.spec.whatwg.org": HostRule( + default=("Core", "DOM: Notifications"), + ), + "encoding.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "compression.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "console.spec.whatwg.org": HostRule( + default=("DevTools", "Console"), + ), + "fullscreen.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "webidl.spec.whatwg.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "compat.spec.whatwg.org": HostRule( + default=("Core", "CSS Parsing and Computation"), + ), + "httpwg.org": HostRule( + default=("Core", "Networking: HTTP"), + sub_rules=[ + PathRule(r"/specs/rfc9842", default=("Core", "Networking: Cache")), + ], + ), + "www.rfc-editor.org": HostRule( + default=("Core", "Networking"), + ), + "w3c-fedid.github.io": HostRule( + default=("Core", "DOM: Credential Management"), + ), + "w3c-cg.github.io": HostRule( + default=None, + sub_rules=[ + PathRule(r"/web-nfc", default=("Core", "DOM: Device Interfaces")), + ], + ), + "webbluetoothcg.github.io": HostRule( + default=("Core", "DOM: Device Interfaces"), + ), + "wicg.github.io": HostRule( + default=("Core", "DOM: Core & HTML"), + sub_rules=[ + PathRule( + r"/attribution-reporting", default=("Core", "Privacy: Anti-Tracking") + ), + PathRule(r"/first-party-sets", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/turtledove", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/shared-storage", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/manifest-incubations", default=("Firefox", "General")), + PathRule(r"/web-app-launch", default=("Firefox", "General")), + PathRule(r"/get-installed-related-apps", default=("Firefox", "General")), + PathRule( + r"/(local-network-access|private-network-access|netinfo|savedata)", + default=("Core", "DOM: Networking"), + ), + PathRule(r"/ua-client-hints", default=("Core", "Networking: HTTP")), + PathRule( + r"/(is-input-pending|performance-measure-memory|js-self-profiling|scheduling-apis)", + default=("Core", "DOM: Performance APIs"), + ), + PathRule(r"/layout-instability", default=("Core", "Layout")), + PathRule(r"/storage-buckets", default=("Core", "Storage: Quota Manager")), + PathRule( + r"/(background-fetch|background-sync|periodic-background-sync|content-index)", + default=("Core", "DOM: Service Workers"), + ), + PathRule(r"/sanitizer-api", default=("Core", "DOM: Security")), + PathRule(r"/signature-based-sri", default=("Core", "DOM: Security")), + PathRule(r"/webcrypto-secure-curves", default=("Core", "DOM: Security")), + PathRule(r"/web-otp", default=("Core", "DOM: Web Authentication")), + PathRule( + r"/(serial|webhid|webusb|shape-detection-api|idle-detection)", + default=("Core", "DOM: Device Interfaces"), + ), + PathRule( + r"/keyboard-(lock|map)", + default=("Core", "DOM: UI Events & Focus Handling"), + ), + PathRule( + r"/ink-enhancement", default=("Core", "DOM: UI Events & Focus Handling") + ), + PathRule(r"/entries-api", default=("Core", "DOM: File")), + PathRule(r"/eyedropper-api", default=("Core", "DOM: Core & HTML")), + PathRule(r"/local-font-access", default=("Core", "Layout: Text and Fonts")), + PathRule( + r"/(crash-reporting|deprecation-reporting|intervention-reporting)", + default=("Core", "DOM: Core & HTML"), + ), + PathRule(r"/nav-speculation", default=("Core", "DOM: Navigation")), + PathRule(r"/scroll-to-text-fragment", default=("Core", "DOM: Navigation")), + PathRule(r"/video-rvfc", default=("Core", "Audio/Video: Playback")), + PathRule( + r"/document-picture-in-picture", default=("Core", "DOM: Core & HTML") + ), + PathRule(r"/controls-list", default=("Core", "Audio/Video: Playback")), + PathRule(r"/file-system-access", default=("WebExtensions", "General")), + PathRule(r"/digital-goods", default=("Core", "DOM: Web Payments")), + PathRule(r"/anonymous-iframe", default=("Core", "DOM: Core & HTML")), + PathRule(r"/fenced-frame", default=("Core", "Privacy: Anti-Tracking")), + PathRule(r"/portals", default=("Core", "DOM: Core & HTML")), + PathRule(r"/PEPC", default=("Core", "DOM: Core & HTML")), + PathRule(r"/window-controls-overlay", default=("Firefox", "General")), + PathRule(r"/observable", default=("Core", "DOM: Core & HTML")), + PathRule(r"/page-lifecycle", default=("Core", "DOM: Core & HTML")), + PathRule(r"/permissions-request", default=("Core", "DOM: Core & HTML")), + ], + ), + "privacycg.github.io": HostRule( + default=("Core", "Privacy: Anti-Tracking"), + ), + "patcg-individual-drafts.github.io": HostRule( + default=("Core", "Privacy: Anti-Tracking"), + ), + "www.iso.org": HostRule( + default=None, + sub_rules=[ + PathRule(r"/standard/", default=("Core", "Graphics: ImageLib")), + ], + ), + "jpeg.org": HostRule( + default=("Core", "Graphics: ImageLib"), + ), + "aomediacodec.github.io": HostRule( + default=("Core", "Graphics: ImageLib"), + ), + "svgwg.org": HostRule( + default=("Core", "SVG"), + ), + "webmachinelearning.github.io": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "screen-share.github.io": HostRule( + default=("Core", "WebRTC: Audio/Video"), + ), + "open-ui.org": HostRule( + default=("Core", "DOM: Core & HTML"), + ), + "www.w3.org": HostRule( + default=None, + sub_rules=[ + PathRule(r"/TR/device-memory", default=("Core", "DOM: Core & HTML")), + PathRule(r"/TR/.*feature-policy", default=("Core", "DOM: Core & HTML")), + PathRule(r"/TR/webnn", default=("Core", "DOM: Core & HTML")), + PathRule(r"/TR/SVG", default=("Core", "SVG")), + PathRule( + r"/TR/.*selectors-", default=("Core", "CSS Parsing and Computation") + ), + PathRule( + r"/TR/DOM-Level-2-Style", default=("Core", "DOM: CSS Object Model") + ), + PathRule(r"/TR/.*html5", default=("Core", "DOM: Core & HTML")), + PathRule(r"/Graphics/GIF", default=("Core", "Graphics: ImageLib")), + ], + ), +} + + +def map_spec(url: str) -> RuleMatch | None: + """Return the most specific :class:`Match` for ``url``, or ``None``.""" + parsed = urlsplit(url) + host_rule = RULES.get(parsed.hostname or "") + if host_rule is None: + return None + return host_rule.get(parsed) + + +def map_spec_urls(urls: Iterable[str]) -> ProductComponent: + """Pick a plausible (product, component) for a feature with given spec URLs.""" + best_match: RuleMatch | None = None + for url in urls: + rule_match = map_spec(url) + if rule_match is None: + continue + if best_match is None or rule_match > best_match: + best_match = rule_match + if best_match is None: + return "Core", "General" + return best_match.product, best_match.component diff --git a/templates/web_platform_features.html b/templates/web_platform_features.html index 1eba74e32..c4ddc259d 100644 --- a/templates/web_platform_features.html +++ b/templates/web_platform_features.html @@ -10,7 +10,7 @@ - {% for i, (bugid, summary, changes, whiteboard, user_story) in enumerate(data) -%} + {% for i, (bugid, summary, change_type, changes, whiteboard, user_story) in enumerate(data) -%} @@ -19,84 +19,146 @@ {{ summary | e }} - + {% elif change_type == "create" %} +

+ Created bug: +

+ {% endif -%} + + + {% endfor -%} +