Support for creating new web-feature bugs

bugbot/rules/web_platform_features.py

@@ -13,17 +13,21 @@
     Any,
     Generic,
     Iterable,
+    Literal,
     Mapping,
     MutableMapping,
     Optional,
     Sequence,
     TypeVar,
+    cast,
 )
 from urllib import parse
 
 from google.cloud import bigquery
+from libmozdata.bugzilla import Bugzilla
+from requests.exceptions import HTTPError
 
-from bugbot import gcp
+from bugbot import gcp, logger, spec_mapping, utils
 from bugbot.bzcleaner import Bug, BzCleaner
 
 Json = None | str | int | float | Sequence["Json"] | Mapping[str, "Json"]
@@ -87,6 +91,43 @@ def __bool__(self) -> bool:
         return bool(self.add or self.remove)
 
 
+@dataclass
+class BugzillaNewBug:
+    """Representation of a new bug to be created"""
+
+    summary: str
+    product: str
+    component: str
+    description: str
+    type: Literal["defect"] | Literal["enhancement"] | Literal["task"]
+    version: str = "unspecified"
+    keywords: Optional[list[str]] = None
+    whiteboard: Optional[str] = None
+    see_also: Optional[list[str]] = None
+    user_story: Optional[str] = None
+    url: Optional[str] = None
+
+    def to_json(self) -> Mapping[str, Json]:
+        rv: dict[str, Json] = {
+            "summary": self.summary,
+            "product": self.product,
+            "component": self.component,
+            "description": self.description,
+            "version": self.version,
+            "type": self.type,
+        }
+        for value, name in [
+            (self.whiteboard, "status_whiteboard"),
+            (self.keywords, "keywords"),
+            (self.see_also, "see_also"),
+            (self.user_story, "cf_user_story"),
+            (self.url, "url"),
+        ]:
+            if value is not None:
+                rv[name] = value
+        return rv
+
+
 @dataclass
 class BugzillaUpdate:
     """Representation of bug changes for use with the Bugzilla ReST API"""
@@ -287,11 +328,36 @@ class FeatureData:
     supported_browsers: set[str]
     sp_issue: Optional[int]
     spec_url: set[str]
+    name: Optional[str] = None
+    description: Optional[str] = None
 
     def is_supported(self) -> bool:
         return {"firefox", "firefox_android"}.issubset(self.supported_browsers)
 
 
+def feature_keywords(feature: FeatureData) -> set[str]:
+    rv = set()
+    if {"chrome", "chrome_android"}.issubset(feature.supported_browsers):
+        rv.add("parity-chrome")
+    if {"safari", "safari_ios"}.issubset(feature.supported_browsers):
+        rv.add("parity-safari")
+    return rv
+
+
+def feature_links(feature: FeatureData) -> set[str]:
+    links = set(
+        [
+            f"https://web-platform-dx.github.io/web-features-explorer/features/{feature.feature}/"
+        ]
+    )
+    if feature.sp_issue is not None:
+        links.add(
+            f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}"
+        )
+    links |= feature.spec_url
+    return links
+
+
 @dataclass
 class FeatureBug:
     """Bug that represents a web-feature"""
@@ -313,26 +379,16 @@ def expected_keywords(self) -> set[str]:
             rv.add("web-feature")
         if not self.is_supported():
             for feature in self.features.values():
-                if {"chrome", "chrome_android"}.issubset(feature.supported_browsers):
-                    rv.add("parity-chrome")
-                if {"safari", "safari_ios"}.issubset(feature.supported_browsers):
-                    rv.add("parity-safari")
+                rv |= feature_keywords(feature)
         return rv
 
     def missing_keywords(self) -> set[str]:
         return self.expected_keywords().difference(self.keywords)
 
     def expected_links(self) -> set[str]:
         links = set()
-        for feature_name, feature in self.features.items():
-            links.add(
-                f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/"
-            )
-            if feature.sp_issue is not None:
-                links.add(
-                    f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}"
-                )
-            links |= feature.spec_url
+        for feature in self.features.values():
+            links |= feature_links(feature)
         return links
 
     def missing_links(self) -> set[str]:
@@ -366,6 +422,78 @@ def remove_links(self) -> set[str]:
 _DataType = TypeVar("_DataType")
 
 
+class CreateRule(ABC, Generic[_DataType]):
+    """Rule for creating new bugs based on BigQuery data"""
+
+    def __init__(self, client: bigquery.Client):
+        self.client = client
+
+    @abstractmethod
+    def get_data(self) -> _DataType:
+        ...
+
+    @abstractmethod
+    def create(self, data: _DataType) -> Mapping[str, BugzillaNewBug]:
+        ...
+
+    def run(self) -> Mapping[str, BugzillaNewBug]:
+        data: _DataType = self.get_data()
+        return self.create(data)
+
+
+class FirefoxOnlyMissing(CreateRule):
+    def get_data(self) -> list[FeatureData]:
+        query = """
+SELECT
+  features.feature,
+  features.name,
+  features.description,
+  (SELECT ARRAY_AGG(browser) FROM UNNEST(features.support)) AS supported_browsers,
+  features.spec as spec_url,
+  sp_mozilla.issue as sp_issue
+FROM `web_features.features_latest` AS features
+LEFT JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs ON
+  features.feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature"))
+LEFT JOIN `standards_positions.mozilla_standards_positions` AS sp_mozilla ON
+  sp_mozilla.web_feature = features.feature
+WHERE
+  "safari" in UNNEST(features.support.browser) AND
+  "chrome" IN UNNEST(features.support.browser) AND
+  "firefox" NOT IN UNNEST(features.support.browser) AND
+  bugs.number IS NULL
+"""
+        return [
+            FeatureData(
+                feature=row.feature,
+                spec_url=set(row.spec_url),
+                supported_browsers=set(row.supported_browsers),
+                sp_issue=row.sp_issue,
+                name=row.name,
+                description=row.description,
+            )
+            for row in self.client.query(query)
+        ]
+
+    def create(self, data: list[FeatureData]) -> Mapping[str, BugzillaNewBug]:
+        rv = {}
+        for feature in data:
+            product, component = spec_mapping.map_spec_urls(feature.spec_url)
+            spec_url = feature.spec_url.pop()
+            rv[feature.feature] = BugzillaNewBug(
+                summary=f"[meta] Implement {feature.name}",
+                product=product,
+                component=component,
+                description=f"Implement {feature.name}:\n{feature.description}",
+                type="enhancement",
+                keywords=["web-feature"] + list(feature_keywords(feature)),
+                user_story=f"web-feature: {feature.feature}",
+                url=spec_url,
+                see_also=[item for item in feature_links(feature) if item != spec_url],
+            )
+
+        return rv
+
+
 class UpdateRule(ABC, Generic[_DataType]):
     """Rule for updating bugs based on BigQuery data"""
 
@@ -598,6 +726,8 @@ def update(
 class WebPlatformFeatures(BzCleaner):
     def __init__(self) -> None:
         super().__init__()
+        self.create_bugs: dict[str, BugzillaNewBug] = {}
+        self.bugs_created: dict[int, BugzillaNewBug] = {}
         self.bug_updates: dict[int, FeatureBugUpdate] = defaultdict(FeatureBugUpdate)
 
     def description(self) -> str:
@@ -610,24 +740,106 @@ def has_default_products(self) -> bool:
         return False
 
     def columns(self) -> list[str]:
-        return ["id", "summary", "changes", "whiteboard", "user_story"]
+        return ["id", "summary", "change_type", "changes", "whiteboard", "user_story"]
+
+    def get_bugs(
+        self,
+        date: str = "today",
+        bug_ids: list[int] = [],
+        chunk_size: Optional[int] = None,
+    ) -> dict[str, Mapping[str, Any]]:
+        bugs = super().get_bugs(date, bug_ids, chunk_size)
+
+        if self.create_bugs:
+            bugs_for_features = self.get_feature_bugs(set(self.create_bugs.keys()))
+        else:
+            bugs_for_features = {}
+
+        for i, (feature, bug) in enumerate(self.create_bugs.items()):
+            if feature in bugs_for_features:
+                # A bug was already created for this feature
+                continue
+
+            if self.dryrun or self.test_mode:
+                response = {"id": i}
+                logger.info(
+                    f"A bug '{bug.summary}` would be created with:\n{bug.to_json()}",
+                )
+            else:
+                try:
+                    response = utils.create_bug(cast(dict, bug.to_json()))
+                except HTTPError:
+                    logger.error(
+                        f"Failed to create bug '{bug.summary}'",
+                    )
+                    continue
+
+            bug_id = response["id"]
+            assert isinstance(bug_id, int)
+            self.bugs_created[bug_id] = bug
+            bugs[str(bug_id)] = {
+                "id": bug_id,
+                "summary": bug.summary,
+                "url": bug.url,
+                "see_also": bug.component,
+                "keywords": bug.keywords,
+                "whiteboard": bug.whiteboard,
+                "cf_user_story": bug.user_story,
+                "status": "NEW",
+                "resolution": "",
+                "change_type": "create",
+                "changes": bug,
+                "user_story": bug.user_story,
+            }
+
+        return bugs
+
+    def get_feature_bugs(self, features: set[str]) -> Mapping[str, int]:
+        """Get the list of existing bugs for specific features"""
+        data: dict[str, int] = {}
+
+        def handler(bug: Mapping[str, Any], data: dict[str, int]) -> None:
+            for _, key, value in parse_user_story(bug["cf_user_story"]):
+                if key == "web-feature":
+                    if value in features:
+                        data[value] = bug["id"]
+
+        Bugzilla(
+            {
+                "keywords": "web-feature",
+                "keywords_type": "allwords",
+                "f1": "cf_user_story",
+                "o1": "substring",
+                "v1": "web-feature",
+                "f2": "cf_user_story",
+                "o2": "anywordssubstr",
+                "v2": ",".join(features),
+            },
+            bugdata=data,
+            bughandler=handler,
+        ).wait()
+
+        return data
 
     def handle_bug(self, bug: Bug, data: dict[str, Any]) -> Optional[Bug]:
         bug_id_str = str(bug["id"])
         bug_id_int = int(bug["id"])
 
-        if bug_id_int not in self.bug_updates:
-            return None
-        bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug)
+        if bug_id_int in self.bug_updates:
+            bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug)
+            if not bugzilla_update:
+                return None
 
-        if bugzilla_update:
             self.autofix_changes[bug_id_str] = bugzilla_update.to_json()
             data[bug_id_str] = {
+                "change_type": "update",
                 "changes": bugzilla_update,
                 "whiteboard": bug["whiteboard"],
                 "user_story": bug["cf_user_story"],
             }
             return bug
+        elif bug_id_int in self.bugs_created:
+            return bug
 
         return None
 
@@ -648,6 +860,10 @@ def get_bz_params(self, date: str) -> dict[str, str | int | list[str] | list[int
     def get_bug_updates(self) -> None:
         project = "moz-fx-dev-dschubert-wckb"
         client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"])
+
+        for create_rule in [FirefoxOnlyMissing(client)]:
+            self.create_bugs.update(create_rule.run())
+
         for update_rule in [
             FeatureRenames(client),
             InvalidFeatures(client),