add debug statements and check add backup patches if they weren't tested

scripts/create_patches_files.py

@@ -92,6 +92,28 @@ def create_files(
         f.write(str(artifact_names))
 
 
+def check_series_is_interesting(patch: Dict[str, Any]):
+    """
+    Grep the series mbox file for key terms/email addresses when
+    the individual patch mbox is not found. Example
+    https://github.com/ewlu/gcc-precommit-ci/actions/runs/14332231733/job/40170680741
+    """
+    r = requests.get(patch["series"][0]["mbox"], timeout=300)  # 5 minutes
+    r.encoding = r.apparent_encoding
+    series_mbox = r.text.lower()
+    print(f"series_mbox link: {patch['series'][0]['mbox']}")
+    print(f"'riscv' in series_mbox check: {'riscv' in series_mbox}")
+    print(f"'risc-v' in series_mbox check: {'risc-v' in series_mbox}")
+    print(
+        f"'patchworks-ci@rivosinc.com' in series_mbox check: {'patchworks-ci@rivosinc.com' in series_mbox}"
+    )
+    return (
+        "riscv" in series_mbox
+        or "risc-v" in series_mbox
+        or "patchworks-ci@rivosinc.com" in series_mbox
+    )
+
+
 def interesting_patch(patch: Dict[str, Any]):
     """Grep the patch mbox file for key terms/email addresses."""
     r = requests.get(patch["mbox"], timeout=300)  # 5 minutes
@@ -101,11 +123,26 @@ def interesting_patch(patch: Dict[str, Any]):
 
     # Search for riscv, risc-v, patchworks-ci@rivosinc.com
     # mbox is already lowercased
-    return (
-        "riscv" in patch_mbox
-        or "risc-v" in patch_mbox
-        or "patchworks-ci@rivosinc.com" in patch_mbox
+    print(f"patch mbox link: {patch['mbox']}")
+    print(f"'riscv' in patch_mbox check: {'riscv' in patch_mbox}")
+    print(f"'risc-v' in patch_mbox check: {'risc-v' in patch_mbox}")
+    print(
+        f"'patchworks-ci@rivosinc.com' in patch_mbox check: {'patchworks-ci@rivosinc.com' in patch_mbox}"
     )
+    if "404: file not found - patchwork" in patch_mbox:
+        # Skip for now. Will require more work to ensure that the correct link
+        # is used. Since interesting_patch is called by parse_patches, by
+        # returning the check_series_is_interesting result, we then need to
+        # change the links in parse_patches to use the series mbox link.
+        # The assert will also make it more apparent that something went wrong.
+        assert False, f"Patch mbox link {patch['mbox']} not found."
+        return check_series_is_interesting(patch)
+    else:
+        return (
+            "riscv" in patch_mbox
+            or "risc-v" in patch_mbox
+            or "patchworks-ci@rivosinc.com" in patch_mbox
+        )
 
 
 def parse_patches(
@@ -117,6 +154,15 @@ def parse_patches(
     all_download_links: DefaultDict[str, List[List[str]]] = defaultdict(list)
     riscv_patchworks_links: DefaultDict[str, List[List[str]]] = defaultdict(list)
     all_patchworks_links: DefaultDict[str, List[List[str]]] = defaultdict(list)
+
+    # these patch check links are primarily used for identifying patches that for
+    # some reason did not run during their corresponding patchwork run and are found
+    # during the backup -> start period. If this is the case, we need to check
+    # to see if the found patches have already been tested in the start -> end period.
+    # If they have, we can skip them.
+    riscv_patch_check_links: DefaultDict[str, List[List[str]]] = defaultdict(list)
+    all_patch_check_links: DefaultDict[str, List[List[str]]] = defaultdict(list)
+
     series_name: Dict[str, str] = {}
     series_url: Dict[str, str] = {}
 
@@ -134,6 +180,7 @@ def parse_patches(
             all_patchworks_links[found_series].append(
                 [f"{patch['name']}\t{patch['web_url']}\t{patch['id']}\n"]
             )
+            all_patch_check_links[found_series].append([patch["checks"] + "\n"])
         else:
             prev_download_link = list(all_download_links[found_series][-1])
             prev_download_link.append(patch["mbox"] + "\n")
@@ -143,6 +190,9 @@ def parse_patches(
                 f"{patch['name']}\t{patch['web_url']}\t{patch['id']}\n"
             )
             all_patchworks_links[found_series].append(prev_patchworks_link)
+            prev_patch_check_link = list(all_patch_check_links[found_series][-1])
+            prev_patch_check_link.append(patch["checks"] + "\n")
+            all_patch_check_links[found_series].append(prev_patch_check_link)
 
         if patch_id is None and (interesting_patch(patch) or all_patches):
             print(f"Patch {patch['name']} is an interesting patch")
@@ -152,6 +202,9 @@ def parse_patches(
             riscv_patchworks_links[found_series].append(
                 all_patchworks_links[found_series][-1]
             )
+            riscv_patch_check_links[found_series].append(
+                all_patch_check_links[found_series][-1]
+            )
 
         # Old check, used for asserts
         if patch_id is None and (
@@ -169,6 +222,9 @@ def parse_patches(
             riscv_patchworks_links[found_series].append(
                 all_patchworks_links[found_series][-1]
             )
+            riscv_patch_check_links[found_series].append(
+                all_patch_check_links[found_series][-1]
+            )
 
         if found_series not in series_name:
             if patch["series"][0]["name"] is None:
@@ -186,10 +242,13 @@ def parse_patches(
     download_links = [item for sublist in download_links for item in sublist]
     patchworks_links = list(riscv_patchworks_links.values())
     patchworks_links = [item for sublist in patchworks_links for item in sublist]
+    patch_check_links = list(riscv_patch_check_links.values())
+    patch_check_links = [item for sublist in patch_check_links for item in sublist]
 
     if all_patches:
         assert len(riscv_download_links) == len(all_download_links)
         assert len(riscv_patchworks_links) == len(all_patchworks_links)
+        assert len(riscv_patch_check_links) == len(all_patch_check_links)
 
     for patch_links in riscv_title_patch_links:
         assert any(
@@ -200,7 +259,19 @@ def parse_patches(
             patch_links == links for links in patchworks_links
         ), f"Expected match (risc-v/riscv in title): {patch_links} not in patchworks links: {patchworks_links}"
 
-    return series_name, series_url, riscv_download_links, riscv_patchworks_links
+    print(f"Returning the following dictionaries:")
+    print(f"series_name: {json.dumps(series_name, indent=4)}\n")
+    print(f"series_url: {json.dumps(series_url, indent=4)}\n")
+    print(f"riscv_download_links: {json.dumps(riscv_download_links, indent=4)}\n")
+    print(f"riscv_patchworks_links: {json.dumps(riscv_patchworks_links, indent=4)}\n")
+    print(f"riscv_patch_check_links: {json.dumps(riscv_patch_check_links, indent=4)}\n")
+    return (
+        series_name,
+        series_url,
+        riscv_download_links,
+        riscv_patchworks_links,
+        riscv_patch_check_links,
+    )
 
 
 def make_api_request(url: str):
@@ -236,13 +307,13 @@ def get_single_patch_info(url: str, patch_id: Union[str, None] = None):
 def get_single_patch(patch_id: str):
     url = f"https://patchwork.sourceware.org/api/1.3/patches/{patch_id}"
 
-    series_name, series_url, download_links, patchwork_links = get_single_patch_info(
-        url, patch_id
+    series_name, series_url, download_links, patchwork_links, _patch_check_links = (
+        get_single_patch_info(url, patch_id)
     )
 
-    print("creating download links")
+    print("creating download links for single patch")
     create_files(series_name, series_url, download_links, "./patch_urls")
-    print("creating patchworks links")
+    print("creating patchworks links for single patch")
     create_files(series_name, series_url, patchwork_links, "./patchworks_metadata")
 
 
@@ -262,19 +333,19 @@ def get_patches_file(file_path: str):
     super_patchwork_links: Dict[str, List[List[str]]] = {}
     for patch in patch_nums:
         url = f"https://patchwork.sourceware.org/api/1.3/patches/{patch}"
-        series_name, series_url, download_links, patchwork_links = (
+        series_name, series_url, download_links, patchwork_links, _patch_check_links = (
             get_single_patch_info(url)
         )
         super_series_name.update(series_name)
         super_series_url.update(series_url)
         super_download_links.update(download_links)
         super_patchwork_links.update(patchwork_links)
 
-    print("creating download links")
+    print("creating download links for provided patches file")
     create_files(
         super_series_name, super_series_url, super_download_links, "./patch_urls"
     )
-    print("creating patchworks links")
+    print("creating patchworks links for provided patches file")
     create_files(
         super_series_name,
         super_series_url,
@@ -284,16 +355,48 @@ def get_patches_file(file_path: str):
 
 
 def get_overlap_dict(
-    download: Dict[str, List[List[str]]], early: Dict[str, List[List[str]]]
+    download: Dict[str, List[List[str]]],
+    early: Dict[str, List[List[str]]],
+    early_patch_check_links: Dict[str, List[List[str]]],
 ):
+    print("checking for overlap between early and download")
+    print(f"early: {json.dumps(early, indent=4)}\n")
+    print(f"download: {json.dumps(download, indent=4)}\n")
+    print(f"early_patch_check_links: {json.dumps(early_patch_check_links, indent=4)}\n")
     overlap = set(early.keys()).intersection(set(download.keys()))
+    print(f"overlap: {overlap}")
     if len(overlap) != 0:
         print("found overlap, downloading sections")
         for series in overlap:
             download[series] = [
                 list(early[series][-1]) + list(val) for val in download[series]
             ]
 
+    else:
+        print("No overlap found. Checking if earlier patches were run")
+        for series, links in early_patch_check_links.items():
+            for index, link in enumerate(
+                links[-1]
+            ):  # links[-1] contains all of links for the series
+                check = make_api_request(link.strip())
+                if check == [] or "toolchain-ci" not in json.dumps(check):
+                    print(
+                        f"Early patch has not been run. Including in download, {early[series][index]}"
+                    )
+                    if series not in download:
+                        download[series] = []
+                    download[series].append(early[series][index])
+                else:
+                    print(
+                        f"Early patch has been run. Skipping {early[series][-1][index].strip()}"
+                    )
+                    print(
+                        f"check == []: {check == []}, 'toolchain-ci' in json.dumps(check): {'toolchain-ci' in json.dumps(check)}"
+                    )
+
+    print(
+        f"after checking overlapping values, download: {json.dumps(download, indent=4)}\n"
+    )
     return download
 
 
@@ -324,22 +427,48 @@ def get_multiple_patches(
 
     print(all_patches)
 
-    series_name, series_url, download_links, patchworks_links = get_patch_info(
-        url.format(project, start, end), all_patches
+    series_name, series_url, download_links, patchworks_links, _patch_check_links = (
+        get_patch_info(url.format(project, start, end), all_patches)
     )
 
+    print(f"parsed information for {start} -> {end}")
+    print(f"series_name: {json.dumps(series_name, indent=4)}\n")
+    print(f"series_url: {json.dumps(series_url, indent=4)}\n")
+    print(f"download_links: {json.dumps(download_links, indent=4)}\n")
+    print(f"patchworks_links: {json.dumps(patchworks_links, indent=4)}\n")
+
     (
         _early_series_name,
         _early_series_url,
         early_download_links,
         early_patchworks_links,
+        early_patch_check_links,
     ) = get_patch_info(url.format(project, backup, start), all_patches)
 
-    print("creating download links")
-    new_download_links = get_overlap_dict(download_links, early_download_links)
+    print(f"parsed information for {backup} -> {start}")
+    print(f"early_series_name: {json.dumps(_early_series_name, indent=4)}\n")
+    print(f"early_series_url: {json.dumps(_early_series_url, indent=4)}\n")
+    print(f"early_download_links: {json.dumps(early_download_links, indent=4)}\n")
+    print(f"early_patchworks_links: {json.dumps(early_patchworks_links, indent=4)}\n")
+    print(f"early_patch_check_links: {json.dumps(early_patch_check_links, indent=4)}\n")
+
+    print("creating download links for multiple patches")
+    new_download_links = get_overlap_dict(
+        download_links, early_download_links, early_patch_check_links
+    )
+    if len(series_name) == 0 and len(new_download_links) != 0:
+        # start -> end period has no patches, but backup -> start period does and since
+        # len(new_download_links) != 0, we know that the previous patches were not run
+        print("No series found in start -> end period, using early series information")
+        series_name = _early_series_name
+        series_url = _early_series_url
+        print(f"series_name: {json.dumps(series_name, indent=4)}\n")
+        print(f"series_url: {json.dumps(series_url, indent=4)}\n")
     create_files(series_name, series_url, new_download_links, "./patch_urls")
-    print("creating patchworks links")
-    new_patchworks_links = get_overlap_dict(patchworks_links, early_patchworks_links)
+    print("creating patchworks links for multiple patches")
+    new_patchworks_links = get_overlap_dict(
+        patchworks_links, early_patchworks_links, early_patch_check_links
+    )
     create_files(series_name, series_url, new_patchworks_links, "./patchworks_metadata")
 
 

scripts/download_artifacts.py

@@ -238,10 +238,10 @@ def get_possible_artifact_names(prefix: str) -> List[str]:
     # Weekly arch extensions included since rv64gcv_zv* doesn't
     # exist without a prefix
     frequent_prefix = [
-            "",
-            "coord_",
-            "release_14_",
-            "release_15_",
+        "",
+        "coord_",
+        "release_14_",
+        "release_15_",
     ]
     if prefix in frequent_prefix:
         return get_frequent_names(prefix)

test/pytests/test_scripts/test_create_patches_files.py

@@ -10,7 +10,7 @@ def test_riscv_body():
     # https://patchwork.sourceware.org/project/gcc/patch/20240611061943.698499-1-pan2.li@intel.com/
     # patch with riscv in the body of the message
     url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&before=2024-06-11T06:20:00&since=2024-06-11T06:15:00"
-    series_name, series_url, download_links, patchworks_links = get_patch_info(url, False)
+    series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False)
 
     assert len(download_links.items()) == 1
     assert len(patchworks_links.items()) == 1
@@ -20,7 +20,7 @@ def test_riscv_title():
     # https://patchwork.sourceware.org/project/gcc/patch/20240701091303.1968994-1-pan2.li@intel.com/
     # patch with riscv in the title
     url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-07-01T09:10:00&before=2024-07-01T09:15:00"
-    series_name, series_url, download_links, patchworks_links = get_patch_info(url, False)
+    series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False)
 
     assert len(download_links.items()) == 1
     assert len(patchworks_links.items()) == 1
@@ -30,15 +30,15 @@ def test_non_riscv():
     # https://patchwork.sourceware.org/project/gcc/patch/20240624135510.3509497-1-pan2.li@intel.com/
     # patch without riscv in the body of the message
     url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-06-24T13:55:00&before=2024-06-24T14:00:00"
-    series_name, series_url, download_links, patchworks_links = get_patch_info(url, False)
+    series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False)
 
     assert len(download_links.items()) == 0
     assert len(patchworks_links.items()) == 0
 
 def test_no_patches():
     # No patches were sent in this time range
     url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-06-24T13:50:00&before=2024-06-24T13:51:00"
-    _series_name, _series_url, download_links, patchworks_links = get_patch_info(url, False)
+    _series_name, _series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False)
 
     assert len(download_links) == 0
     assert len(patchworks_links) == 0