From 03714f5227cdd11e0237066526c51a6bab5efa39 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 3 Jul 2026 12:06:25 +0200 Subject: [PATCH] ci-doctor: separate product severity from failure frequency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report previously conflated two dimensions under one CRITICAL/HIGH/ MEDIUM/LOW badge computed purely from affected-job count: a one-job severity-5 release blocker rendered LOW while a five-job flake rendered CRITICAL — actively misleading triage. Issues now carry both: severity (max of the analysis' 1-5 product- impact rubric, rendered S1–S5) and frequency (the count-based label, rendered as a second chip). Old summary files with only the label still render, so doctor-refresh on an existing workdir keeps working. --- plugins/shared/scripts/aggregate.py | 11 +++++- plugins/shared/scripts/create-report.py | 49 +++++++++++++++++++++---- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/plugins/shared/scripts/aggregate.py b/plugins/shared/scripts/aggregate.py index a8500f85..14611692 100755 --- a/plugins/shared/scripts/aggregate.py +++ b/plugins/shared/scripts/aggregate.py @@ -25,7 +25,13 @@ from parse import parse_structured_summary, group_by_signature -def classify_severity(group): +def classify_frequency(group): + """How often this issue hit, by affected-job count. + + Deliberately separate from `severity` (the 1-5 product-impact rubric + from the analysis): a one-job release blocker is severe but not + frequent; a five-job flake is frequent but not severe. + """ count = len(group) if count >= 5: return "CRITICAL" @@ -83,7 +89,8 @@ def _build_issues_from_jobs(jobs): "number": i, "title": rep["error_signature"], "job_count": len(group), - "severity": classify_severity(group), + "severity": max(j["severity"] for j in group), + "frequency": classify_frequency(group), "failure_type": failure_type, "root_cause": rep.get("root_cause", ""), "next_steps": rep.get("remediation", ""), diff --git a/plugins/shared/scripts/create-report.py b/plugins/shared/scripts/create-report.py index 4c0a330b..be3ac708 100755 --- a/plugins/shared/scripts/create-report.py +++ b/plugins/shared/scripts/create-report.py @@ -115,6 +115,11 @@ .severity-medium { background: #fff3cd; color: #856404; } .severity-low { background: #d4edda; color: #155724; } .severity-critical { background: #721c24; color: #fff; } + .sev-5 { background: #721c24; color: #fff; } + .sev-4 { background: #f8d7da; color: #721c24; } + .sev-3 { background: #ffe5d0; color: #7d4a10; } + .sev-2 { background: #fff3cd; color: #856404; } + .sev-1 { background: #e2e3e5; color: #41464b; } .ftype-badge { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.75em; font-weight: 700; text-transform: uppercase; } .ftype-test { background: #cce5ff; color: #004085; } .ftype-build { background: #e2d5f1; color: #4a235a; } @@ -990,6 +995,40 @@ def _render_confidence_badge(issue): f' title="Root cause analysis confidence">{conf}') +def _severity_cell(issue): + """Severity (1-5 product impact) + frequency (job count) badges. + + New summaries carry severity as an int and frequency as a label; + older summaries have only the count-based label in `severity` — + render whichever is present. + """ + freq_labels = ("HIGH", "MEDIUM", "LOW", "CRITICAL") + sev = issue.get("severity", "") + parts = [] + if isinstance(sev, int) or (isinstance(sev, str) and sev.isdigit()): + n = max(1, min(5, int(sev))) + parts.append(f'S{n}') + else: + label = str(sev).upper() or "UNKNOWN" + css = f"severity-{label.lower()}" if label in freq_labels else "" + parts.append(f'{label}') + freq = str(issue.get("frequency", "")).upper() + if freq in freq_labels: + parts.append(f'{freq}') + return " ".join(parts) + + +def _is_critical_issue(issue): + if str(issue.get("frequency", "")).upper() == "CRITICAL": + return True + sev = issue.get("severity", "") + if isinstance(sev, int) or (isinstance(sev, str) and sev.isdigit()): + return int(sev) >= 5 + return str(sev).upper() == "CRITICAL" + + def _render_investigation(issue): """Render scenario chips, causal chain, and analysis gaps for an issue. @@ -1227,7 +1266,7 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): ) total = rdata["total_failed"] - has_critical = any(i.get("severity", "").upper() == "CRITICAL" for i in rdata["issues"]) + has_critical = any(_is_critical_issue(i) for i in rdata["issues"]) badge = _badge_class(total, has_critical) b = rdata["breakdown"] @@ -1253,8 +1292,6 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): for issue in rdata["issues"]: bug_match = match_issue_to_bugs(issue["title"], bug_candidates) jc = issue["job_count"] - sev = issue.get("severity", "UNKNOWN").upper() - sev_css = f"severity-{sev.lower()}" if sev in ("HIGH", "MEDIUM", "LOW", "CRITICAL") else "" ftype = issue.get("failure_type", "test") ftype_label = "INFRA" if ftype == "infrastructure" else ftype.upper() ftype_css = "ftype-infra" if ftype == "infrastructure" else f"ftype-{ftype}" @@ -1264,7 +1301,7 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): dates_attr = f' data-dates="{" ".join(job_dates)}"' if job_dates else "" anchor_id = f'release-{_e(version)}-{issue["number"]}' lines.append(f' ') - lines.append(f' {sev}') + lines.append(f' {_severity_cell(issue)}') lines.append(f' {ftype_label}') lines.append(f' {_e(issue["title"])}') lines.append(f' {jobs_label}') @@ -1418,8 +1455,6 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): for issue in analysis["issues"]: bug_match = match_issue_to_bugs(issue.get("title", ""), bug_candidates) jc = issue["job_count"] - sev = issue.get("severity", "UNKNOWN").upper() - sev_css = f"severity-{sev.lower()}" if sev in ("HIGH", "MEDIUM", "LOW", "CRITICAL") else "" ftype = issue.get("failure_type", "test") ftype_label = "INFRA" if ftype == "infrastructure" else ftype.upper() ftype_css = "ftype-infra" if ftype == "infrastructure" else f"ftype-{ftype}" @@ -1427,7 +1462,7 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): anchor_id = f'pr-{pr["number"]}-{issue["number"]}' lines.append(f' ') - lines.append(f' {sev}') + lines.append(f' {_severity_cell(issue)}') lines.append(f' {ftype_label}') lines.append(f' {_e(issue["title"])}') lines.append(f' {jobs_label}')