From 41812c95f8796515dd1ca85fcaf17582bef9bf36 Mon Sep 17 00:00:00 2001 From: Edwin Lu Date: Tue, 5 Aug 2025 14:42:13 -0700 Subject: [PATCH 1/2] add debug statements and check add backup patches if they weren't tested --- scripts/create_patches_files.py | 141 +++++++++++++++--- .../test_scripts/test_create_patches_files.py | 8 +- 2 files changed, 127 insertions(+), 22 deletions(-) diff --git a/scripts/create_patches_files.py b/scripts/create_patches_files.py index 4c63c053349..77d734fd5c3 100644 --- a/scripts/create_patches_files.py +++ b/scripts/create_patches_files.py @@ -92,6 +92,26 @@ def create_files( f.write(str(artifact_names)) +def check_series_is_interesting(patch: Dict[str, Any]): + """ + Grep the series mbox file for key terms/email addresses when + the individual patch mbox is not found. Example + https://github.com/ewlu/gcc-precommit-ci/actions/runs/14332231733/job/40170680741 + """ + r = requests.get(patch["series"][0]["mbox"], timeout=300) # 5 minutes + r.encoding = r.apparent_encoding + series_mbox = r.text.lower() + print(f"series_mbox link: {patch['series'][0]['mbox']}") + print(f"'riscv' in series_mbox check: {'riscv' in series_mbox}") + print(f"'risc-v' in series_mbox check: {'risc-v' in series_mbox}") + print(f"'patchworks-ci@rivosinc.com' in series_mbox check: {'patchworks-ci@rivosinc.com' in series_mbox}") + return ( + "riscv" in series_mbox + or "risc-v" in series_mbox + or "patchworks-ci@rivosinc.com" in series_mbox + ) + + def interesting_patch(patch: Dict[str, Any]): """Grep the patch mbox file for key terms/email addresses.""" r = requests.get(patch["mbox"], timeout=300) # 5 minutes @@ -101,11 +121,24 @@ def interesting_patch(patch: Dict[str, Any]): # Search for riscv, risc-v, patchworks-ci@rivosinc.com # mbox is already lowercased - return ( - "riscv" in patch_mbox - or "risc-v" in patch_mbox - or "patchworks-ci@rivosinc.com" in patch_mbox - ) + print(f"patch mbox link: {patch['mbox']}") + print(f"'riscv' in patch_mbox check: {'riscv' in patch_mbox}") + print(f"'risc-v' in patch_mbox check: {'risc-v' in patch_mbox}") + print(f"'patchworks-ci@rivosinc.com' in patch_mbox check: {'patchworks-ci@rivosinc.com' in patch_mbox}") + if "404: file not found - patchwork" in patch_mbox: + # Skip for now. Will require more work to ensure that the correct link + # is used. Since interesting_patch is called by parse_patches, by + # returning the check_series_is_interesting result, we then need to + # change the links in parse_patches to use the series mbox link. + # The assert will also make it more apparent that something went wrong. + assert False, f"Patch mbox link {patch['mbox']} not found." + return check_series_is_interesting(patch) + else: + return ( + "riscv" in patch_mbox + or "risc-v" in patch_mbox + or "patchworks-ci@rivosinc.com" in patch_mbox + ) def parse_patches( @@ -117,6 +150,15 @@ def parse_patches( all_download_links: DefaultDict[str, List[List[str]]] = defaultdict(list) riscv_patchworks_links: DefaultDict[str, List[List[str]]] = defaultdict(list) all_patchworks_links: DefaultDict[str, List[List[str]]] = defaultdict(list) + + # these patch check links are primarily used for identifying patches that for + # some reason did not run during their corresponding patchwork run and are found + # during the backup -> start period. If this is the case, we need to check + # to see if the found patches have already been tested in the start -> end period. + # If they have, we can skip them. + riscv_patch_check_links: DefaultDict[str, List[List[str]]] = defaultdict(list) + all_patch_check_links: DefaultDict[str, List[List[str]]] = defaultdict(list) + series_name: Dict[str, str] = {} series_url: Dict[str, str] = {} @@ -134,6 +176,7 @@ def parse_patches( all_patchworks_links[found_series].append( [f"{patch['name']}\t{patch['web_url']}\t{patch['id']}\n"] ) + all_patch_check_links[found_series].append([patch["checks"] + "\n"]) else: prev_download_link = list(all_download_links[found_series][-1]) prev_download_link.append(patch["mbox"] + "\n") @@ -143,6 +186,9 @@ def parse_patches( f"{patch['name']}\t{patch['web_url']}\t{patch['id']}\n" ) all_patchworks_links[found_series].append(prev_patchworks_link) + prev_patch_check_link = list(all_patch_check_links[found_series][-1]) + prev_patch_check_link.append(patch["checks"] + "\n") + all_patch_check_links[found_series].append(prev_patch_check_link) if patch_id is None and (interesting_patch(patch) or all_patches): print(f"Patch {patch['name']} is an interesting patch") @@ -152,6 +198,9 @@ def parse_patches( riscv_patchworks_links[found_series].append( all_patchworks_links[found_series][-1] ) + riscv_patch_check_links[found_series].append( + all_patch_check_links[found_series][-1] + ) # Old check, used for asserts if patch_id is None and ( @@ -169,6 +218,9 @@ def parse_patches( riscv_patchworks_links[found_series].append( all_patchworks_links[found_series][-1] ) + riscv_patch_check_links[found_series].append( + all_patch_check_links[found_series][-1] + ) if found_series not in series_name: if patch["series"][0]["name"] is None: @@ -186,10 +238,13 @@ def parse_patches( download_links = [item for sublist in download_links for item in sublist] patchworks_links = list(riscv_patchworks_links.values()) patchworks_links = [item for sublist in patchworks_links for item in sublist] + patch_check_links = list(riscv_patch_check_links.values()) + patch_check_links = [item for sublist in patch_check_links for item in sublist] if all_patches: assert len(riscv_download_links) == len(all_download_links) assert len(riscv_patchworks_links) == len(all_patchworks_links) + assert len(riscv_patch_check_links) == len(all_patch_check_links) for patch_links in riscv_title_patch_links: assert any( @@ -200,7 +255,13 @@ def parse_patches( patch_links == links for links in patchworks_links ), f"Expected match (risc-v/riscv in title): {patch_links} not in patchworks links: {patchworks_links}" - return series_name, series_url, riscv_download_links, riscv_patchworks_links + print(f"Returning the following dictionaries:") + print(f"series_name: {json.dumps(series_name, indent=4)}\n") + print(f"series_url: {json.dumps(series_url, indent=4)}\n") + print(f"riscv_download_links: {json.dumps(riscv_download_links, indent=4)}\n") + print(f"riscv_patchworks_links: {json.dumps(riscv_patchworks_links, indent=4)}\n") + print(f"riscv_patch_check_links: {json.dumps(riscv_patch_check_links, indent=4)}\n") + return series_name, series_url, riscv_download_links, riscv_patchworks_links, riscv_patch_check_links def make_api_request(url: str): @@ -236,13 +297,13 @@ def get_single_patch_info(url: str, patch_id: Union[str, None] = None): def get_single_patch(patch_id: str): url = f"https://patchwork.sourceware.org/api/1.3/patches/{patch_id}" - series_name, series_url, download_links, patchwork_links = get_single_patch_info( + series_name, series_url, download_links, patchwork_links, _patch_check_links = get_single_patch_info( url, patch_id ) - print("creating download links") + print("creating download links for single patch") create_files(series_name, series_url, download_links, "./patch_urls") - print("creating patchworks links") + print("creating patchworks links for single patch") create_files(series_name, series_url, patchwork_links, "./patchworks_metadata") @@ -262,7 +323,7 @@ def get_patches_file(file_path: str): super_patchwork_links: Dict[str, List[List[str]]] = {} for patch in patch_nums: url = f"https://patchwork.sourceware.org/api/1.3/patches/{patch}" - series_name, series_url, download_links, patchwork_links = ( + series_name, series_url, download_links, patchwork_links, _patch_check_links = ( get_single_patch_info(url) ) super_series_name.update(series_name) @@ -270,11 +331,11 @@ def get_patches_file(file_path: str): super_download_links.update(download_links) super_patchwork_links.update(patchwork_links) - print("creating download links") + print("creating download links for provided patches file") create_files( super_series_name, super_series_url, super_download_links, "./patch_urls" ) - print("creating patchworks links") + print("creating patchworks links for provided patches file") create_files( super_series_name, super_series_url, @@ -284,9 +345,16 @@ def get_patches_file(file_path: str): def get_overlap_dict( - download: Dict[str, List[List[str]]], early: Dict[str, List[List[str]]] + download: Dict[str, List[List[str]]], + early: Dict[str, List[List[str]]], + early_patch_check_links: Dict[str, List[List[str]]] ): + print("checking for overlap between early and download") + print(f"early: {json.dumps(early, indent=4)}\n") + print(f"download: {json.dumps(download, indent=4)}\n") + print(f"early_patch_check_links: {json.dumps(early_patch_check_links, indent=4)}\n") overlap = set(early.keys()).intersection(set(download.keys())) + print(f"overlap: {overlap}") if len(overlap) != 0: print("found overlap, downloading sections") for series in overlap: @@ -294,6 +362,21 @@ def get_overlap_dict( list(early[series][-1]) + list(val) for val in download[series] ] + else: + print("No overlap found. Checking if earlier patches were run") + for series, links in early_patch_check_links.items(): + for index, link in enumerate(links[-1]): # links[-1] contains all of links for the series + check = make_api_request(link.strip()) + if check == [] or "toolchain-ci" not in json.dumps(check): + print(f"Early patch has not been run. Including in download, {early[series][index]}") + if series not in download: + download[series] = [] + download[series].append(early[series][index]) + else: + print(f"Early patch has been run. Skipping {early[series][-1][index].strip()}") + print(f"check == []: {check == []}, 'toolchain-ci' in json.dumps(check): {'toolchain-ci' in json.dumps(check)}") + + print(f"after checking overlapping values, download: {json.dumps(download, indent=4)}\n") return download @@ -324,22 +407,44 @@ def get_multiple_patches( print(all_patches) - series_name, series_url, download_links, patchworks_links = get_patch_info( + series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info( url.format(project, start, end), all_patches ) + print(f"parsed information for {start} -> {end}") + print(f"series_name: {json.dumps(series_name, indent=4)}\n") + print(f"series_url: {json.dumps(series_url, indent=4)}\n") + print(f"download_links: {json.dumps(download_links, indent=4)}\n") + print(f"patchworks_links: {json.dumps(patchworks_links, indent=4)}\n") + ( _early_series_name, _early_series_url, early_download_links, early_patchworks_links, + early_patch_check_links, ) = get_patch_info(url.format(project, backup, start), all_patches) - print("creating download links") - new_download_links = get_overlap_dict(download_links, early_download_links) + print(f"parsed information for {backup} -> {start}") + print(f"early_series_name: {json.dumps(_early_series_name, indent=4)}\n") + print(f"early_series_url: {json.dumps(_early_series_url, indent=4)}\n") + print(f"early_download_links: {json.dumps(early_download_links, indent=4)}\n") + print(f"early_patchworks_links: {json.dumps(early_patchworks_links, indent=4)}\n") + print(f"early_patch_check_links: {json.dumps(early_patch_check_links, indent=4)}\n") + + print("creating download links for multiple patches") + new_download_links = get_overlap_dict(download_links, early_download_links, early_patch_check_links) + if len(series_name) == 0 and len(new_download_links) != 0: + # start -> end period has no patches, but backup -> start period does and since + # len(new_download_links) != 0, we know that the previous patches were not run + print("No series found in start -> end period, using early series information") + series_name = _early_series_name + series_url = _early_series_url + print(f"series_name: {json.dumps(series_name, indent=4)}\n") + print(f"series_url: {json.dumps(series_url, indent=4)}\n") create_files(series_name, series_url, new_download_links, "./patch_urls") - print("creating patchworks links") - new_patchworks_links = get_overlap_dict(patchworks_links, early_patchworks_links) + print("creating patchworks links for multiple patches") + new_patchworks_links = get_overlap_dict(patchworks_links, early_patchworks_links, early_patch_check_links) create_files(series_name, series_url, new_patchworks_links, "./patchworks_metadata") diff --git a/test/pytests/test_scripts/test_create_patches_files.py b/test/pytests/test_scripts/test_create_patches_files.py index eeecb6f154e..155bdcedd5a 100644 --- a/test/pytests/test_scripts/test_create_patches_files.py +++ b/test/pytests/test_scripts/test_create_patches_files.py @@ -10,7 +10,7 @@ def test_riscv_body(): # https://patchwork.sourceware.org/project/gcc/patch/20240611061943.698499-1-pan2.li@intel.com/ # patch with riscv in the body of the message url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&before=2024-06-11T06:20:00&since=2024-06-11T06:15:00" - series_name, series_url, download_links, patchworks_links = get_patch_info(url, False) + series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links.items()) == 1 assert len(patchworks_links.items()) == 1 @@ -20,7 +20,7 @@ def test_riscv_title(): # https://patchwork.sourceware.org/project/gcc/patch/20240701091303.1968994-1-pan2.li@intel.com/ # patch with riscv in the title url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-07-01T09:10:00&before=2024-07-01T09:15:00" - series_name, series_url, download_links, patchworks_links = get_patch_info(url, False) + series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links.items()) == 1 assert len(patchworks_links.items()) == 1 @@ -30,7 +30,7 @@ def test_non_riscv(): # https://patchwork.sourceware.org/project/gcc/patch/20240624135510.3509497-1-pan2.li@intel.com/ # patch without riscv in the body of the message url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-06-24T13:55:00&before=2024-06-24T14:00:00" - series_name, series_url, download_links, patchworks_links = get_patch_info(url, False) + series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links.items()) == 0 assert len(patchworks_links.items()) == 0 @@ -38,7 +38,7 @@ def test_non_riscv(): def test_no_patches(): # No patches were sent in this time range url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-06-24T13:50:00&before=2024-06-24T13:51:00" - _series_name, _series_url, download_links, patchworks_links = get_patch_info(url, False) + _series_name, _series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links) == 0 assert len(patchworks_links) == 0 From 7d0bffb3fff6d265787a6b3ff4e3f16d41863c18 Mon Sep 17 00:00:00 2001 From: Edwin Lu Date: Tue, 5 Aug 2025 15:44:03 -0700 Subject: [PATCH 2/2] precommit formatting --- scripts/create_patches_files.py | 56 +++++++++++++++++++++++---------- scripts/download_artifacts.py | 8 ++--- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/scripts/create_patches_files.py b/scripts/create_patches_files.py index 77d734fd5c3..78f5381e1e4 100644 --- a/scripts/create_patches_files.py +++ b/scripts/create_patches_files.py @@ -104,7 +104,9 @@ def check_series_is_interesting(patch: Dict[str, Any]): print(f"series_mbox link: {patch['series'][0]['mbox']}") print(f"'riscv' in series_mbox check: {'riscv' in series_mbox}") print(f"'risc-v' in series_mbox check: {'risc-v' in series_mbox}") - print(f"'patchworks-ci@rivosinc.com' in series_mbox check: {'patchworks-ci@rivosinc.com' in series_mbox}") + print( + f"'patchworks-ci@rivosinc.com' in series_mbox check: {'patchworks-ci@rivosinc.com' in series_mbox}" + ) return ( "riscv" in series_mbox or "risc-v" in series_mbox @@ -124,7 +126,9 @@ def interesting_patch(patch: Dict[str, Any]): print(f"patch mbox link: {patch['mbox']}") print(f"'riscv' in patch_mbox check: {'riscv' in patch_mbox}") print(f"'risc-v' in patch_mbox check: {'risc-v' in patch_mbox}") - print(f"'patchworks-ci@rivosinc.com' in patch_mbox check: {'patchworks-ci@rivosinc.com' in patch_mbox}") + print( + f"'patchworks-ci@rivosinc.com' in patch_mbox check: {'patchworks-ci@rivosinc.com' in patch_mbox}" + ) if "404: file not found - patchwork" in patch_mbox: # Skip for now. Will require more work to ensure that the correct link # is used. Since interesting_patch is called by parse_patches, by @@ -261,7 +265,13 @@ def parse_patches( print(f"riscv_download_links: {json.dumps(riscv_download_links, indent=4)}\n") print(f"riscv_patchworks_links: {json.dumps(riscv_patchworks_links, indent=4)}\n") print(f"riscv_patch_check_links: {json.dumps(riscv_patch_check_links, indent=4)}\n") - return series_name, series_url, riscv_download_links, riscv_patchworks_links, riscv_patch_check_links + return ( + series_name, + series_url, + riscv_download_links, + riscv_patchworks_links, + riscv_patch_check_links, + ) def make_api_request(url: str): @@ -297,8 +307,8 @@ def get_single_patch_info(url: str, patch_id: Union[str, None] = None): def get_single_patch(patch_id: str): url = f"https://patchwork.sourceware.org/api/1.3/patches/{patch_id}" - series_name, series_url, download_links, patchwork_links, _patch_check_links = get_single_patch_info( - url, patch_id + series_name, series_url, download_links, patchwork_links, _patch_check_links = ( + get_single_patch_info(url, patch_id) ) print("creating download links for single patch") @@ -347,7 +357,7 @@ def get_patches_file(file_path: str): def get_overlap_dict( download: Dict[str, List[List[str]]], early: Dict[str, List[List[str]]], - early_patch_check_links: Dict[str, List[List[str]]] + early_patch_check_links: Dict[str, List[List[str]]], ): print("checking for overlap between early and download") print(f"early: {json.dumps(early, indent=4)}\n") @@ -365,18 +375,28 @@ def get_overlap_dict( else: print("No overlap found. Checking if earlier patches were run") for series, links in early_patch_check_links.items(): - for index, link in enumerate(links[-1]): # links[-1] contains all of links for the series + for index, link in enumerate( + links[-1] + ): # links[-1] contains all of links for the series check = make_api_request(link.strip()) if check == [] or "toolchain-ci" not in json.dumps(check): - print(f"Early patch has not been run. Including in download, {early[series][index]}") + print( + f"Early patch has not been run. Including in download, {early[series][index]}" + ) if series not in download: download[series] = [] download[series].append(early[series][index]) else: - print(f"Early patch has been run. Skipping {early[series][-1][index].strip()}") - print(f"check == []: {check == []}, 'toolchain-ci' in json.dumps(check): {'toolchain-ci' in json.dumps(check)}") - - print(f"after checking overlapping values, download: {json.dumps(download, indent=4)}\n") + print( + f"Early patch has been run. Skipping {early[series][-1][index].strip()}" + ) + print( + f"check == []: {check == []}, 'toolchain-ci' in json.dumps(check): {'toolchain-ci' in json.dumps(check)}" + ) + + print( + f"after checking overlapping values, download: {json.dumps(download, indent=4)}\n" + ) return download @@ -407,8 +427,8 @@ def get_multiple_patches( print(all_patches) - series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info( - url.format(project, start, end), all_patches + series_name, series_url, download_links, patchworks_links, _patch_check_links = ( + get_patch_info(url.format(project, start, end), all_patches) ) print(f"parsed information for {start} -> {end}") @@ -433,7 +453,9 @@ def get_multiple_patches( print(f"early_patch_check_links: {json.dumps(early_patch_check_links, indent=4)}\n") print("creating download links for multiple patches") - new_download_links = get_overlap_dict(download_links, early_download_links, early_patch_check_links) + new_download_links = get_overlap_dict( + download_links, early_download_links, early_patch_check_links + ) if len(series_name) == 0 and len(new_download_links) != 0: # start -> end period has no patches, but backup -> start period does and since # len(new_download_links) != 0, we know that the previous patches were not run @@ -444,7 +466,9 @@ def get_multiple_patches( print(f"series_url: {json.dumps(series_url, indent=4)}\n") create_files(series_name, series_url, new_download_links, "./patch_urls") print("creating patchworks links for multiple patches") - new_patchworks_links = get_overlap_dict(patchworks_links, early_patchworks_links, early_patch_check_links) + new_patchworks_links = get_overlap_dict( + patchworks_links, early_patchworks_links, early_patch_check_links + ) create_files(series_name, series_url, new_patchworks_links, "./patchworks_metadata") diff --git a/scripts/download_artifacts.py b/scripts/download_artifacts.py index 8eee9f14101..eb9b7b91910 100644 --- a/scripts/download_artifacts.py +++ b/scripts/download_artifacts.py @@ -238,10 +238,10 @@ def get_possible_artifact_names(prefix: str) -> List[str]: # Weekly arch extensions included since rv64gcv_zv* doesn't # exist without a prefix frequent_prefix = [ - "", - "coord_", - "release_14_", - "release_15_", + "", + "coord_", + "release_14_", + "release_15_", ] if prefix in frequent_prefix: return get_frequent_names(prefix)