diff --git a/scripts/create_patches_files.py b/scripts/create_patches_files.py index 4c63c053349..78f5381e1e4 100644 --- a/scripts/create_patches_files.py +++ b/scripts/create_patches_files.py @@ -92,6 +92,28 @@ def create_files( f.write(str(artifact_names)) +def check_series_is_interesting(patch: Dict[str, Any]): + """ + Grep the series mbox file for key terms/email addresses when + the individual patch mbox is not found. Example + https://github.com/ewlu/gcc-precommit-ci/actions/runs/14332231733/job/40170680741 + """ + r = requests.get(patch["series"][0]["mbox"], timeout=300) # 5 minutes + r.encoding = r.apparent_encoding + series_mbox = r.text.lower() + print(f"series_mbox link: {patch['series'][0]['mbox']}") + print(f"'riscv' in series_mbox check: {'riscv' in series_mbox}") + print(f"'risc-v' in series_mbox check: {'risc-v' in series_mbox}") + print( + f"'patchworks-ci@rivosinc.com' in series_mbox check: {'patchworks-ci@rivosinc.com' in series_mbox}" + ) + return ( + "riscv" in series_mbox + or "risc-v" in series_mbox + or "patchworks-ci@rivosinc.com" in series_mbox + ) + + def interesting_patch(patch: Dict[str, Any]): """Grep the patch mbox file for key terms/email addresses.""" r = requests.get(patch["mbox"], timeout=300) # 5 minutes @@ -101,11 +123,26 @@ def interesting_patch(patch: Dict[str, Any]): # Search for riscv, risc-v, patchworks-ci@rivosinc.com # mbox is already lowercased - return ( - "riscv" in patch_mbox - or "risc-v" in patch_mbox - or "patchworks-ci@rivosinc.com" in patch_mbox + print(f"patch mbox link: {patch['mbox']}") + print(f"'riscv' in patch_mbox check: {'riscv' in patch_mbox}") + print(f"'risc-v' in patch_mbox check: {'risc-v' in patch_mbox}") + print( + f"'patchworks-ci@rivosinc.com' in patch_mbox check: {'patchworks-ci@rivosinc.com' in patch_mbox}" ) + if "404: file not found - patchwork" in patch_mbox: + # Skip for now. Will require more work to ensure that the correct link + # is used. Since interesting_patch is called by parse_patches, by + # returning the check_series_is_interesting result, we then need to + # change the links in parse_patches to use the series mbox link. + # The assert will also make it more apparent that something went wrong. + assert False, f"Patch mbox link {patch['mbox']} not found." + return check_series_is_interesting(patch) + else: + return ( + "riscv" in patch_mbox + or "risc-v" in patch_mbox + or "patchworks-ci@rivosinc.com" in patch_mbox + ) def parse_patches( @@ -117,6 +154,15 @@ def parse_patches( all_download_links: DefaultDict[str, List[List[str]]] = defaultdict(list) riscv_patchworks_links: DefaultDict[str, List[List[str]]] = defaultdict(list) all_patchworks_links: DefaultDict[str, List[List[str]]] = defaultdict(list) + + # these patch check links are primarily used for identifying patches that for + # some reason did not run during their corresponding patchwork run and are found + # during the backup -> start period. If this is the case, we need to check + # to see if the found patches have already been tested in the start -> end period. + # If they have, we can skip them. + riscv_patch_check_links: DefaultDict[str, List[List[str]]] = defaultdict(list) + all_patch_check_links: DefaultDict[str, List[List[str]]] = defaultdict(list) + series_name: Dict[str, str] = {} series_url: Dict[str, str] = {} @@ -134,6 +180,7 @@ def parse_patches( all_patchworks_links[found_series].append( [f"{patch['name']}\t{patch['web_url']}\t{patch['id']}\n"] ) + all_patch_check_links[found_series].append([patch["checks"] + "\n"]) else: prev_download_link = list(all_download_links[found_series][-1]) prev_download_link.append(patch["mbox"] + "\n") @@ -143,6 +190,9 @@ def parse_patches( f"{patch['name']}\t{patch['web_url']}\t{patch['id']}\n" ) all_patchworks_links[found_series].append(prev_patchworks_link) + prev_patch_check_link = list(all_patch_check_links[found_series][-1]) + prev_patch_check_link.append(patch["checks"] + "\n") + all_patch_check_links[found_series].append(prev_patch_check_link) if patch_id is None and (interesting_patch(patch) or all_patches): print(f"Patch {patch['name']} is an interesting patch") @@ -152,6 +202,9 @@ def parse_patches( riscv_patchworks_links[found_series].append( all_patchworks_links[found_series][-1] ) + riscv_patch_check_links[found_series].append( + all_patch_check_links[found_series][-1] + ) # Old check, used for asserts if patch_id is None and ( @@ -169,6 +222,9 @@ def parse_patches( riscv_patchworks_links[found_series].append( all_patchworks_links[found_series][-1] ) + riscv_patch_check_links[found_series].append( + all_patch_check_links[found_series][-1] + ) if found_series not in series_name: if patch["series"][0]["name"] is None: @@ -186,10 +242,13 @@ def parse_patches( download_links = [item for sublist in download_links for item in sublist] patchworks_links = list(riscv_patchworks_links.values()) patchworks_links = [item for sublist in patchworks_links for item in sublist] + patch_check_links = list(riscv_patch_check_links.values()) + patch_check_links = [item for sublist in patch_check_links for item in sublist] if all_patches: assert len(riscv_download_links) == len(all_download_links) assert len(riscv_patchworks_links) == len(all_patchworks_links) + assert len(riscv_patch_check_links) == len(all_patch_check_links) for patch_links in riscv_title_patch_links: assert any( @@ -200,7 +259,19 @@ def parse_patches( patch_links == links for links in patchworks_links ), f"Expected match (risc-v/riscv in title): {patch_links} not in patchworks links: {patchworks_links}" - return series_name, series_url, riscv_download_links, riscv_patchworks_links + print(f"Returning the following dictionaries:") + print(f"series_name: {json.dumps(series_name, indent=4)}\n") + print(f"series_url: {json.dumps(series_url, indent=4)}\n") + print(f"riscv_download_links: {json.dumps(riscv_download_links, indent=4)}\n") + print(f"riscv_patchworks_links: {json.dumps(riscv_patchworks_links, indent=4)}\n") + print(f"riscv_patch_check_links: {json.dumps(riscv_patch_check_links, indent=4)}\n") + return ( + series_name, + series_url, + riscv_download_links, + riscv_patchworks_links, + riscv_patch_check_links, + ) def make_api_request(url: str): @@ -236,13 +307,13 @@ def get_single_patch_info(url: str, patch_id: Union[str, None] = None): def get_single_patch(patch_id: str): url = f"https://patchwork.sourceware.org/api/1.3/patches/{patch_id}" - series_name, series_url, download_links, patchwork_links = get_single_patch_info( - url, patch_id + series_name, series_url, download_links, patchwork_links, _patch_check_links = ( + get_single_patch_info(url, patch_id) ) - print("creating download links") + print("creating download links for single patch") create_files(series_name, series_url, download_links, "./patch_urls") - print("creating patchworks links") + print("creating patchworks links for single patch") create_files(series_name, series_url, patchwork_links, "./patchworks_metadata") @@ -262,7 +333,7 @@ def get_patches_file(file_path: str): super_patchwork_links: Dict[str, List[List[str]]] = {} for patch in patch_nums: url = f"https://patchwork.sourceware.org/api/1.3/patches/{patch}" - series_name, series_url, download_links, patchwork_links = ( + series_name, series_url, download_links, patchwork_links, _patch_check_links = ( get_single_patch_info(url) ) super_series_name.update(series_name) @@ -270,11 +341,11 @@ def get_patches_file(file_path: str): super_download_links.update(download_links) super_patchwork_links.update(patchwork_links) - print("creating download links") + print("creating download links for provided patches file") create_files( super_series_name, super_series_url, super_download_links, "./patch_urls" ) - print("creating patchworks links") + print("creating patchworks links for provided patches file") create_files( super_series_name, super_series_url, @@ -284,9 +355,16 @@ def get_patches_file(file_path: str): def get_overlap_dict( - download: Dict[str, List[List[str]]], early: Dict[str, List[List[str]]] + download: Dict[str, List[List[str]]], + early: Dict[str, List[List[str]]], + early_patch_check_links: Dict[str, List[List[str]]], ): + print("checking for overlap between early and download") + print(f"early: {json.dumps(early, indent=4)}\n") + print(f"download: {json.dumps(download, indent=4)}\n") + print(f"early_patch_check_links: {json.dumps(early_patch_check_links, indent=4)}\n") overlap = set(early.keys()).intersection(set(download.keys())) + print(f"overlap: {overlap}") if len(overlap) != 0: print("found overlap, downloading sections") for series in overlap: @@ -294,6 +372,31 @@ def get_overlap_dict( list(early[series][-1]) + list(val) for val in download[series] ] + else: + print("No overlap found. Checking if earlier patches were run") + for series, links in early_patch_check_links.items(): + for index, link in enumerate( + links[-1] + ): # links[-1] contains all of links for the series + check = make_api_request(link.strip()) + if check == [] or "toolchain-ci" not in json.dumps(check): + print( + f"Early patch has not been run. Including in download, {early[series][index]}" + ) + if series not in download: + download[series] = [] + download[series].append(early[series][index]) + else: + print( + f"Early patch has been run. Skipping {early[series][-1][index].strip()}" + ) + print( + f"check == []: {check == []}, 'toolchain-ci' in json.dumps(check): {'toolchain-ci' in json.dumps(check)}" + ) + + print( + f"after checking overlapping values, download: {json.dumps(download, indent=4)}\n" + ) return download @@ -324,22 +427,48 @@ def get_multiple_patches( print(all_patches) - series_name, series_url, download_links, patchworks_links = get_patch_info( - url.format(project, start, end), all_patches + series_name, series_url, download_links, patchworks_links, _patch_check_links = ( + get_patch_info(url.format(project, start, end), all_patches) ) + print(f"parsed information for {start} -> {end}") + print(f"series_name: {json.dumps(series_name, indent=4)}\n") + print(f"series_url: {json.dumps(series_url, indent=4)}\n") + print(f"download_links: {json.dumps(download_links, indent=4)}\n") + print(f"patchworks_links: {json.dumps(patchworks_links, indent=4)}\n") + ( _early_series_name, _early_series_url, early_download_links, early_patchworks_links, + early_patch_check_links, ) = get_patch_info(url.format(project, backup, start), all_patches) - print("creating download links") - new_download_links = get_overlap_dict(download_links, early_download_links) + print(f"parsed information for {backup} -> {start}") + print(f"early_series_name: {json.dumps(_early_series_name, indent=4)}\n") + print(f"early_series_url: {json.dumps(_early_series_url, indent=4)}\n") + print(f"early_download_links: {json.dumps(early_download_links, indent=4)}\n") + print(f"early_patchworks_links: {json.dumps(early_patchworks_links, indent=4)}\n") + print(f"early_patch_check_links: {json.dumps(early_patch_check_links, indent=4)}\n") + + print("creating download links for multiple patches") + new_download_links = get_overlap_dict( + download_links, early_download_links, early_patch_check_links + ) + if len(series_name) == 0 and len(new_download_links) != 0: + # start -> end period has no patches, but backup -> start period does and since + # len(new_download_links) != 0, we know that the previous patches were not run + print("No series found in start -> end period, using early series information") + series_name = _early_series_name + series_url = _early_series_url + print(f"series_name: {json.dumps(series_name, indent=4)}\n") + print(f"series_url: {json.dumps(series_url, indent=4)}\n") create_files(series_name, series_url, new_download_links, "./patch_urls") - print("creating patchworks links") - new_patchworks_links = get_overlap_dict(patchworks_links, early_patchworks_links) + print("creating patchworks links for multiple patches") + new_patchworks_links = get_overlap_dict( + patchworks_links, early_patchworks_links, early_patch_check_links + ) create_files(series_name, series_url, new_patchworks_links, "./patchworks_metadata") diff --git a/scripts/download_artifacts.py b/scripts/download_artifacts.py index 8eee9f14101..eb9b7b91910 100644 --- a/scripts/download_artifacts.py +++ b/scripts/download_artifacts.py @@ -238,10 +238,10 @@ def get_possible_artifact_names(prefix: str) -> List[str]: # Weekly arch extensions included since rv64gcv_zv* doesn't # exist without a prefix frequent_prefix = [ - "", - "coord_", - "release_14_", - "release_15_", + "", + "coord_", + "release_14_", + "release_15_", ] if prefix in frequent_prefix: return get_frequent_names(prefix) diff --git a/test/pytests/test_scripts/test_create_patches_files.py b/test/pytests/test_scripts/test_create_patches_files.py index eeecb6f154e..155bdcedd5a 100644 --- a/test/pytests/test_scripts/test_create_patches_files.py +++ b/test/pytests/test_scripts/test_create_patches_files.py @@ -10,7 +10,7 @@ def test_riscv_body(): # https://patchwork.sourceware.org/project/gcc/patch/20240611061943.698499-1-pan2.li@intel.com/ # patch with riscv in the body of the message url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&before=2024-06-11T06:20:00&since=2024-06-11T06:15:00" - series_name, series_url, download_links, patchworks_links = get_patch_info(url, False) + series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links.items()) == 1 assert len(patchworks_links.items()) == 1 @@ -20,7 +20,7 @@ def test_riscv_title(): # https://patchwork.sourceware.org/project/gcc/patch/20240701091303.1968994-1-pan2.li@intel.com/ # patch with riscv in the title url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-07-01T09:10:00&before=2024-07-01T09:15:00" - series_name, series_url, download_links, patchworks_links = get_patch_info(url, False) + series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links.items()) == 1 assert len(patchworks_links.items()) == 1 @@ -30,7 +30,7 @@ def test_non_riscv(): # https://patchwork.sourceware.org/project/gcc/patch/20240624135510.3509497-1-pan2.li@intel.com/ # patch without riscv in the body of the message url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-06-24T13:55:00&before=2024-06-24T14:00:00" - series_name, series_url, download_links, patchworks_links = get_patch_info(url, False) + series_name, series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links.items()) == 0 assert len(patchworks_links.items()) == 0 @@ -38,7 +38,7 @@ def test_non_riscv(): def test_no_patches(): # No patches were sent in this time range url = "https://patchwork.sourceware.org/api/1.3/patches/?order=date&project=6&since=2024-06-24T13:50:00&before=2024-06-24T13:51:00" - _series_name, _series_url, download_links, patchworks_links = get_patch_info(url, False) + _series_name, _series_url, download_links, patchworks_links, _patch_check_links = get_patch_info(url, False) assert len(download_links) == 0 assert len(patchworks_links) == 0