From 976fceb21ff3911665209ff93e67a6e10594ab70 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 14:13:55 +0530 Subject: [PATCH 1/8] Add z-stream testing support to CI doctor pipeline - Add zstream-trigger.sh for detecting and triggering z-stream e2e jobs - Integrate z-stream into doctor prepare/finalize using unified release-* naming - Render z-stream sections in report with patch version labels (e.g., 4.19.3) - Add Z-Stream checkbox to toggle between regular and z-stream views - Make report component-agnostic via COMPONENT_REPOS mapping - GCS-based state persistence for CI step-registry trigger script Co-Authored-By: Claude Opus 4.6 --- .claude-plugin/marketplace.json | 2 +- plugins/lvms-ci/.claude-plugin/plugin.json | 2 +- plugins/lvms-ci/scripts/zstream-trigger.sh | 483 +++++++++++++++++++++ plugins/lvms-ci/skills/doctor/SKILL.md | 26 +- plugins/shared/scripts/create-report.py | 160 ++++++- plugins/shared/scripts/doctor.sh | 111 ++++- 6 files changed, 751 insertions(+), 33 deletions(-) create mode 100755 plugins/lvms-ci/scripts/zstream-trigger.sh diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 11deda42..315e184e 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -58,7 +58,7 @@ "name": "lvms-ci", "source": "./plugins/lvms-ci", "description": "LVMS CI Automation", - "version": "1.1.1" + "version": "1.2.0" }, { "name": "mcp-atlassian", diff --git a/plugins/lvms-ci/.claude-plugin/plugin.json b/plugins/lvms-ci/.claude-plugin/plugin.json index bb14f33c..ab37f590 100644 --- a/plugins/lvms-ci/.claude-plugin/plugin.json +++ b/plugins/lvms-ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "lvms-ci", "description": "LVMS CI Automation", - "version": "1.1.1", + "version": "1.2.0", "author": { "name": "knarra" }, diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh new file mode 100755 index 00000000..45e1f71f --- /dev/null +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -0,0 +1,483 @@ +#!/usr/bin/bash +set -euo pipefail + +# Trigger LVMS z-stream e2e tests via Gangway API. +# +# For each configured release, checks GitHub for open z-stream PRs on +# openshift/lvm-operator, resolves the catalog image digest from the PR +# diff via the quay.io API, and triggers existing nightly e2e jobs with +# MULTISTAGE_PARAM_OVERRIDE_LVM_INDEX_IMAGE set to the resolved digest. +# +# Skips releases with no open PR or where the digest hasn't changed since +# the last run (tracked via a state file). +# +# Usage: +# zstream-trigger.sh [--releases 4.14,4.16,...] [--dry-run] [--workdir /path] +# [--force] [--token-file /path/to/token] +# +# Environment: +# GITHUB_TOKEN Optional. GitHub PAT for higher API rate limits. +# MY_APPCI_TOKEN Required (unless --dry-run). Gangway API bearer token. + +RELEASES="4.14,4.16,4.18,4.19,4.20,4.21" +DRY_RUN=false +FORCE=false +WORKDIR="" +TOKEN_FILE="" + +GANGWAY_API="https://gangway-ci.apps.ci.l2s4.p1.openshiftapps.com" +GITHUB_API="https://api.github.com" +QUAY_API="https://quay.io/api/v1" +QUAY_REPO="redhat-user-workloads/logical-volume-manag-tenant/lvm-operator-catalog" +STATE_DIR="${HOME}/.cache/lvms-zstream" +STATE_FILE="${STATE_DIR}/last-tested.json" + +NIGHTLY_JOBS=( + "e2e-aws-sno-qe-integration-tests" + "e2e-aws-sno-arm-qe-integration-tests" + "e2e-aws-mno-qe-integration-tests" + "e2e-aws-mno-arm-qe-integration-tests" + "e2e-baremetalds-sno-dualstack-qe-integration-tests" + "e2e-baremetalds-mno-dualstack-qe-integration-tests" +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +log() { echo "[$(date +%H:%M:%S)] $*" >&2; } +info() { log "INFO $*"; } +warn() { log "WARN $*"; } +err() { log "ERROR $*"; } + +github_curl() { + local -a headers=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + headers+=(-H "Authorization: token ${GITHUB_TOKEN}") + fi + curl -sSL --connect-timeout 10 --max-time 30 "${headers[@]}" "$@" +} + +usage() { + cat >&2 <) + --token-file F Read MY_APPCI_TOKEN from file instead of environment + -h, --help Show this help +EOF + exit 1 +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- + +while [[ ${#} -gt 0 ]]; do + case "${1}" in + --releases) [[ ${#} -ge 2 ]] || { err "--releases requires an argument"; usage; } + RELEASES="${2}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --force) FORCE=true; shift ;; + --workdir) [[ ${#} -ge 2 ]] || { err "--workdir requires an argument"; usage; } + WORKDIR="${2}"; shift 2 ;; + --token-file) [[ ${#} -ge 2 ]] || { err "--token-file requires an argument"; usage; } + TOKEN_FILE="${2}"; shift 2 ;; + -h|--help) usage ;; + *) err "Unknown option: ${1}"; usage ;; + esac +done + +WORKDIR="${WORKDIR:-/tmp/lvms-zstream.$(date +%y%m%d)}" +mkdir -p "${WORKDIR}" + +if [[ -n "${TOKEN_FILE}" && -f "${TOKEN_FILE}" ]]; then + MY_APPCI_TOKEN=$(cat "${TOKEN_FILE}") +fi + +if ! ${DRY_RUN} && [[ -z "${MY_APPCI_TOKEN:-}" ]]; then + err "MY_APPCI_TOKEN is required (set env or use --token-file)" + exit 1 +fi + +# --------------------------------------------------------------------------- +# State management — track last-tested digest per release +# --------------------------------------------------------------------------- + +load_state() { + mkdir -p "${STATE_DIR}" + if [[ ! -f "${STATE_FILE}" ]]; then + echo '{}' > "${STATE_FILE}" + fi +} + +get_last_digest() { + local release="$1" + jq -r --arg r "${release}" '.[$r].digest // ""' "${STATE_FILE}" +} + +get_last_pr() { + local release="$1" + jq -r --arg r "${release}" '.[$r].pr // ""' "${STATE_FILE}" +} + +save_digest() { + local release="$1" digest="$2" pr_number="$3" image="$4" + local tmp + tmp=$(mktemp) + jq --arg r "${release}" \ + --arg d "${digest}" \ + --arg p "${pr_number}" \ + --arg i "${image}" \ + --arg t "$(date -Iseconds)" \ + '.[$r] = {digest: $d, pr: $p, image: $i, tested_at: $t}' \ + "${STATE_FILE}" > "${tmp}" && mv "${tmp}" "${STATE_FILE}" +} + +# --------------------------------------------------------------------------- +# GitHub: find open z-stream PR for a release +# --------------------------------------------------------------------------- + +find_zstream_pr() { + local release="$1" + local major minor + major=$(echo "${release}" | cut -d. -f1) + minor=$(echo "${release}" | cut -d. -f2) + + local prs_json + prs_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls?state=open&per_page=100") + + echo "${prs_json}" | jq -r --arg maj "${major}" --arg min "${minor}" ' + [.[] | select(.title | test(": Release " + $maj + "\\." + $min + "\\."; "i"))] | + if length > 0 then .[0] else empty end | + [.number, .title] | @tsv + ' +} + +# --------------------------------------------------------------------------- +# GitHub: check if a previously tracked PR was merged +# --------------------------------------------------------------------------- + +check_pr_merged() { + local pr_number="$1" + local pr_json + pr_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}") + + local state merged_at + state=$(echo "${pr_json}" | jq -r '.state // ""') + merged_at=$(echo "${pr_json}" | jq -r '.merged_at // ""') + + if [[ "${state}" == "closed" && -n "${merged_at}" && "${merged_at}" != "null" ]]; then + echo "merged" + fi +} + +# --------------------------------------------------------------------------- +# GitHub: extract snapshot name from PR diff +# --------------------------------------------------------------------------- + +extract_snapshot() { + local pr_number="$1" + + local files_json + files_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}/files") + + echo "${files_json}" | jq -r ' + [.[] | select(.filename | contains("catalog")) | .patch // ""] | + join("\n") + ' | { grep -oP '(?<=snapshot: )\S+' || true; } | head -1 +} + +# --------------------------------------------------------------------------- +# Quay: resolve snapshot to digest +# --------------------------------------------------------------------------- + +resolve_snapshot_to_digest() { + local snapshot="$1" + + # Parse version prefix and timestamp from snapshot name + # Format: lvm-operator-catalog-4-16-20260624-033025-000 + local version_prefix date_str time_str + version_prefix=$(echo "${snapshot}" | grep -oP 'lvm-operator-catalog-\d+-\d+') + date_str=$(echo "${snapshot}" | grep -oP '\d{8}(?=-\d{6}-)') + time_str=$(echo "${snapshot}" | grep -oP '(?<=\d{8}-)\d{6}') + + if [[ -z "${version_prefix}" || -z "${date_str}" || -z "${time_str}" ]]; then + err "Cannot parse snapshot name: ${snapshot}" + return 1 + fi + + # Convert snapshot timestamp to epoch for comparison + local snap_year snap_month snap_day snap_hour snap_min snap_sec snap_epoch + snap_year=${date_str:0:4} + snap_month=${date_str:4:2} + snap_day=${date_str:6:2} + snap_hour=${time_str:0:2} + snap_min=${time_str:2:2} + snap_sec=${time_str:4:2} + snap_epoch=$(date -d "${snap_year}-${snap_month}-${snap_day}T${snap_hour}:${snap_min}:${snap_sec}Z" +%s 2>/dev/null || echo 0) + + # Query quay for on-push build-image-index tags matching this version + local tags_json + tags_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ + "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:${version_prefix}-on-push") + + # Find build-image-index tags created after the snapshot timestamp + # Pick the closest one (smallest positive time delta) + local result + result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" ' + [.tags[] + | select(.name | endswith("build-image-index")) + | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) + | .delta = (.tag_epoch - ($snap_epoch | tonumber)) + | select(.delta >= 0 and .delta < 1800) + ] | sort_by(.delta) | .[0] // empty | + [.name, .manifest_digest] | @tsv + ') + + if [[ -z "${result}" ]]; then + err "No matching quay tag found for snapshot ${snapshot}" + return 1 + fi + + local tag_name digest + tag_name=$(echo "${result}" | cut -f1) + digest=$(echo "${result}" | cut -f2) + + info "Resolved: ${snapshot} → ${tag_name} (${digest})" + echo "${digest}" +} + +# --------------------------------------------------------------------------- +# Gangway: trigger a nightly job with image override +# --------------------------------------------------------------------------- + +trigger_job() { + local job_name="$1" image="$2" + + local body + body=$(jq -cn \ + --arg img "${image}" \ + '{job_execution_type: "1", pod_spec_options: {envs: {MULTISTAGE_PARAM_OVERRIDE_LVM_INDEX_IMAGE: $img}}}') + + if ${DRY_RUN}; then + info "[dry-run] would trigger: ${job_name}" + info "[dry-run] image: ${image}" + return 0 + fi + + local http_code + http_code=$(curl -sSL -X POST -o /dev/stderr -w '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${MY_APPCI_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${body}" \ + "${GANGWAY_API}/v1/executions/${job_name}" 2>/dev/null) + + if [[ "${http_code}" == "200" ]]; then + info " triggered: ${job_name}" + return 0 + else + warn " failed (HTTP ${http_code}): ${job_name}" + return 1 + fi +} + +# --------------------------------------------------------------------------- +# Prow: fetch latest run results for nightly jobs +# --------------------------------------------------------------------------- + +fetch_last_runs() { + local release="$1" + local job_prefix="periodic-ci-openshift-lvm-operator-release-${release}-nightly-" + local results="[]" + + for test_name in "${NIGHTLY_JOBS[@]}"; do + local full_job="${job_prefix}${test_name}" + local prow_json + prow_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ + "https://prow.ci.openshift.org/prowjobs.js?omit=annotations,labels,decoration_config,pod_spec&job=${full_job}" 2>/dev/null || true) + + if [[ -z "${prow_json}" ]]; then + results=$(echo "${results}" | jq --arg n "${test_name}" '. + [{name: $n, state: "unknown"}]') + continue + fi + + local entry + entry=$(echo "${prow_json}" | sed 's/^var allBuilds = //' | \ + jq -r --arg n "${test_name}" ' + [.items[] | select(.status.state == "success" or .status.state == "failure" or .status.state == "error" or .status.state == "aborted")] | + if length > 0 then .[0] else null end | + if . then {name: $n, state: .status.state, url: .status.url, started: .status.startTime} else {name: $n, state: "unknown"} end + ' 2>/dev/null || echo "{\"name\": \"${test_name}\", \"state\": \"unknown\"}") + + results=$(echo "${results}" | jq --argjson e "${entry}" '. + [$e]') + done + + echo "${results}" +} + +# --------------------------------------------------------------------------- +# Process a single release +# --------------------------------------------------------------------------- + +process_release() { + local release="$1" + local major minor + major=$(echo "${release}" | cut -d. -f1) + minor=$(echo "${release}" | cut -d. -f2) + + info "--- Release ${release} ---" + + # Find open z-stream PR + local pr_info + pr_info=$(find_zstream_pr "${release}" || true) + + if [[ -z "${pr_info}" ]]; then + # Check if a previously tracked PR was merged + local last_pr + last_pr=$(get_last_pr "${release}") + if [[ -n "${last_pr}" ]]; then + local merge_status + merge_status=$(check_pr_merged "${last_pr}" || true) + if [[ "${merge_status}" == "merged" ]]; then + info "PR #${last_pr} was merged for ${release}." + jq -n --arg r "${release}" --arg pr "${last_pr}" \ + '{($r): {status: "completed", reason: "release completed", pr: ("#" + $pr)}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + fi + + info "No z-stream release PR found for ${release}." + jq -n --arg r "${release}" \ + '{($r): {status: "no_pr", reason: "no new releases"}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + + local pr_number pr_title + pr_number=$(echo "${pr_info}" | cut -f1) + pr_title=$(echo "${pr_info}" | cut -f2) + info "Found PR #${pr_number}: ${pr_title}" + + # Extract snapshot from PR + local snapshot + snapshot=$(extract_snapshot "${pr_number}") + + if [[ -z "${snapshot}" ]]; then + warn "Could not extract catalog snapshot from PR #${pr_number}" + jq -n --arg r "${release}" --arg pr "${pr_number}" \ + '{($r): {status: "error", reason: "no snapshot in PR", pr: ("#" + $pr)}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + + info "Snapshot: ${snapshot}" + + # Resolve to digest + local digest + digest=$(resolve_snapshot_to_digest "${snapshot}" || true) + + if [[ -z "${digest}" ]]; then + warn "Could not resolve snapshot to digest for ${release}" + jq -n --arg r "${release}" --arg pr "${pr_number}" --arg snap "${snapshot}" \ + '{($r): {status: "error", reason: "digest resolution failed", pr: ("#" + $pr), snapshot: $snap}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + + local image="quay.io/${QUAY_REPO}@${digest}" + + # Check if already tested + if ! ${FORCE}; then + local last_digest + last_digest=$(get_last_digest "${release}") + if [[ "${last_digest}" == "${digest}" ]]; then + info "Already tested with same digest. Fetching last run results." + local last_runs + last_runs=$(fetch_last_runs "${release}") + jq -n --arg r "${release}" --arg pr "${pr_number}" --arg title "${pr_title}" --arg img "${image}" --arg snap "${snapshot}" \ + --argjson jobs "${last_runs}" \ + '{($r): {status: "skipped", reason: "same digest", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs: $jobs}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + fi + + # Trigger nightly jobs + local job_prefix="periodic-ci-openshift-lvm-operator-release-${release}-nightly-" + local triggered=0 failed=0 + local triggered_jobs=() + + for test_name in "${NIGHTLY_JOBS[@]}"; do + local full_job="${job_prefix}${test_name}" + if trigger_job "${full_job}" "${image}"; then + triggered=$((triggered + 1)) + triggered_jobs+=("${full_job}") + else + failed=$((failed + 1)) + fi + ${DRY_RUN} || sleep 5 + done + + info "${release}: ${triggered} triggered, ${failed} failed" + + # Update state + if ! ${DRY_RUN} && [[ ${triggered} -gt 0 && ${failed} -eq 0 ]]; then + save_digest "${release}" "${digest}" "${pr_number}" "${image}" + fi + + # Write per-release summary + jq -n \ + --arg r "${release}" \ + --arg pr "${pr_number}" \ + --arg title "${pr_title}" \ + --arg img "${image}" \ + --arg snap "${snapshot}" \ + --argjson triggered "${triggered}" \ + --argjson failed "${failed}" \ + --argjson jobs "$(printf '%s\n' "${triggered_jobs[@]}" | jq -R . | jq -s .)" \ + '{($r): {status: "triggered", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs_triggered: $triggered, jobs_failed: $failed, jobs: $jobs}}' \ + > "${WORKDIR}/zstream-${release}.json" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +main() { + info "=== LVMS Z-Stream Trigger ===" + info "Releases: ${RELEASES}" + info "Workdir: ${WORKDIR}" + ${DRY_RUN} && info "Mode: DRY RUN" + ${FORCE} && info "Mode: FORCE (ignoring last-tested digest)" + + load_state + + IFS=',' read -ra release_list <<< "${RELEASES}" + + for release in "${release_list[@]}"; do + release=$(echo "${release}" | xargs) + process_release "${release}" + done + + # Merge per-release summaries into one file + local summary_file="${WORKDIR}/zstream-summary.json" + jq -s 'add' "${WORKDIR}"/zstream-*.json > "${summary_file}" 2>/dev/null || echo '{}' > "${summary_file}" + + # Print summary + info "" + info "=== Summary ===" + jq -r 'to_entries[] | " \(.key): \(.value.status)\(if .value.reason then " (\(.value.reason))" elif .value.jobs_triggered then " (\(.value.jobs_triggered) jobs)" else "" end)"' \ + "${summary_file}" >&2 + + info "" + info "Summary written to: ${summary_file}" +} + +main diff --git a/plugins/lvms-ci/skills/doctor/SKILL.md b/plugins/lvms-ci/skills/doctor/SKILL.md index c9e929e3..80ab000e 100644 --- a/plugins/lvms-ci/skills/doctor/SKILL.md +++ b/plugins/lvms-ci/skills/doctor/SKILL.md @@ -82,9 +82,11 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa Use the Write tool to save the file. The file must contain the complete analysis report." ``` -3. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. -4. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. -5. When all agents return, immediately proceed to Step 3 in the same turn. Do NOT stop or end your turn between Step 2 and Step 3. +3. If the prepare output contains a `"zstream"` key, process those entries the same way — they use patch versions (e.g., `4.19.3`) as the `` value but follow the identical naming and analysis pattern. + +4. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. +5. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. +6. When all agents return, immediately proceed to Step 3 in the same turn. Do NOT stop or end your turn between Step 2 and Step 3. ### Step 3: Finalize — Aggregate and Generate HTML Report @@ -143,6 +145,24 @@ HTML report generated: /report-lvm-operator-ci-doctor.html - Internet access to fetch job data from Prow/GCS - Bash shell, Python 3 +### Z-Stream Integration (Optional) + +If a `zstream-summary.json` file exists in the workdir and contains releases with `"status": "skipped"` (same digest already tested), the HTML report will include a **Z-Stream** checkbox on the Periodics tab. When checked, it switches the view to show z-stream release status per version (PR info, image digest, triggered jobs). + +To populate z-stream data before generating the report: + +```text +bash plugins/lvms-ci/scripts/zstream-trigger.sh --dry-run --workdir +``` + +This queries GitHub for open z-stream release PRs on `openshift/lvm-operator`, resolves catalog image digests via quay.io, and writes `zstream-summary.json` to the workdir — without triggering any jobs (dry-run mode). + +To actually trigger z-stream e2e jobs (requires `MY_APPCI_TOKEN`): + +```text +bash plugins/lvms-ci/scripts/zstream-trigger.sh --workdir +``` + ## Related Skills - **lvms-ci:prow-job**: Single job analysis (used by Step 2 agents) diff --git a/plugins/shared/scripts/create-report.py b/plugins/shared/scripts/create-report.py index 4c0a330b..ac4a5a4f 100755 --- a/plugins/shared/scripts/create-report.py +++ b/plugins/shared/scripts/create-report.py @@ -43,6 +43,10 @@ "lvm-operator": "LVMS", } +COMPONENT_REPOS = { + "lvm-operator": "openshift/lvm-operator", + "microshift": "openshift/microshift", +} _GRADE_ORDER = {"A": 0, "B": 1, "C": 2, "D": 3, "F": 4} _GRADE_CSS = {"A": "grade-a", "B": "grade-b", "C": "grade-c", "D": "grade-d", "F": "grade-f"} @@ -191,8 +195,13 @@ if (el) el.style.display = el.style.display === 'none' ? 'block' : 'none'; } function filterToday(on) { + var zstreamCheck = document.getElementById('filter-zstream'); + if (on && zstreamCheck && zstreamCheck.checked) { + zstreamCheck.checked = false; + if (typeof filterZstream === 'function') filterZstream(false); + } var today = new Date().toISOString().split('T')[0]; - document.querySelectorAll('#tab-periodics .issue-row').forEach(function(row) { + document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section) .issue-row').forEach(function(row) { var dates = (row.getAttribute('data-dates') || '').split(' '); var show = !on || dates.indexOf(today) !== -1; row.style.display = show ? '' : 'none'; @@ -202,7 +211,7 @@ detail.style.display = show ? '' : 'none'; } }); - document.querySelectorAll('#tab-periodics .release-section').forEach(function(sec) { + document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section)').forEach(function(sec) { var id = sec.id.replace('release-', ''); var rows = sec.querySelectorAll('.issue-row'); var total = 0, bd = {build: 0, test: 0, infra: 0}; @@ -240,6 +249,7 @@ row.style.display = (!on || row.hasAttribute('data-latest')) ? '' : 'none'; }); } + function showGraphTab(btn, paneId) { var container = btn.closest('.perf-graphs'); container.querySelectorAll('.graph-tab-btn').forEach(function(b) { b.classList.remove('active'); }); @@ -1051,7 +1061,7 @@ def _render_index_image(index_info): short = commit[:12] if len(commit) >= 12 else commit lines.append( f' Source Commit: ' - f'{_e(short)}' + f'{_e(short)}' ) if index_info.get("error"): lines.append(f'
Inspect failed: {_e(index_info["error"])}') @@ -1059,8 +1069,11 @@ def _render_index_image(index_info): return "\n".join(lines) + + # Graph workdir — set by main() before rendering _GRAPHS_DIR = None +_COMPONENT = None def _extract_build_id(url): @@ -1203,12 +1216,22 @@ def _render_bug_links(bug_match): # HTML rendering # --------------------------------------------------------------------------- -def render_release_section(version, rdata, bug_candidates, index_info=None): +def render_release_section(version, rdata, bug_candidates, index_info=None, zstream_info=None): + is_zstream = zstream_info is not None + display = version + if is_zstream: + pr_title = zstream_info.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + if m: + display = m.group(1) + css_class = "release-section zstream-section" if is_zstream else "release-section" + style = ' style="display:none"' if is_zstream else "" + if rdata is None: return ( - f'
\n' + f'
\n' '
\n' - f'

Release {_e(version)}

\n' + f'

Release {_e(display)}

\n' ' no data\n' '
\n' "

Analysis failed to produce results.

\n" @@ -1217,9 +1240,9 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): if rdata.get("collection_error"): return ( - f'
\n' + f'
\n' '
\n' - f'

Release {_e(version)}

\n' + f'

Release {_e(display)}

\n' ' collection error\n' '
\n' f' 
Data collection failed: {_e(rdata["collection_error"])}
\n' @@ -1232,14 +1255,17 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): b = rdata["breakdown"] lines = [] - lines.append(f'
') + lines.append(f'
') lines.append('
') - lines.append(f'

Release {_e(version)}🔗

') + lines.append(f'

Release {_e(display)}🔗

') label = "failure" if total == 1 else "failures" lines.append(f' {total} {label}') lines.append("
") - idx_html = _render_index_image(index_info) + if is_zstream: + idx_html = _render_zstream_info(version, zstream_info) + else: + idx_html = _render_index_image(index_info) if idx_html: lines.append(idx_html) @@ -1453,7 +1479,31 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): return "\n".join(toc_lines) + "\n\n" + "\n".join(lines) -def generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error=None, bugs_tab_data=None, images_tab_data=None, index_data=None): +# --------------------------------------------------------------------------- +# Z-Stream sections (rendered inside Periodics tab, toggled via checkbox) +# --------------------------------------------------------------------------- + + +def _render_zstream_info(version, zinfo): + """Render z-stream metadata box (PR, snapshot, image).""" + lines = ['
'] + pr = zinfo.get("pr", "") + image = zinfo.get("image", "") + snapshot = zinfo.get("snapshot", "") + if pr: + pr_display = pr.lstrip("#") + repo = COMPONENT_REPOS.get(_COMPONENT, "openshift/" + (_COMPONENT or "unknown")) + pr_url = f"https://github.com/{repo}/pull/{pr_display}" + lines.append(f' Release PR: #{_e(pr_display)}
') + if snapshot: + lines.append(f' Snapshot: {_e(snapshot)}
') + if image: + lines.append(f' Catalog Image: {_e(image)}
') + lines.append("
") + return "\n".join(lines) + + +def generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error=None, bugs_tab_data=None, images_tab_data=None, index_data=None, zstream_data=None): date_str = timestamp.strftime("%Y-%m-%d") time_str = timestamp.strftime("%Y-%m-%d %H:%M:%S") @@ -1516,10 +1566,58 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p for version, rdata in releases_data.items(): sections.append(render_release_section(version, rdata, all_bug_candidates, _idx.get(version))) + # Append z-stream sections using the same render_release_section path + zstream_checkbox = "" + if zstream_data: + sorted_zversions = sorted( + [zv for zv, zi in zstream_data.items() if zi.get("status") == "skipped"], + key=lambda v: [int(x) for x in v.split(".")], + ) + if sorted_zversions: + zstream_checkbox = ( + ' ' + ) + for zv in sorted_zversions: + zinfo = zstream_data[zv] + empty_bd = {"build": 0, "test": 0, "infrastructure": 0} + rdata = zinfo.get("_analyzed", {"total_failed": 0, "issues": [], "breakdown": empty_bd}) + section_id = f"{zv}-zstream" + pr_title = zinfo.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + display = m.group(1) if m else section_id + total = rdata["total_failed"] if rdata else 0 + b = rdata.get("breakdown", empty_bd) if rdata else empty_bd + fail_word = "failure" if total == 1 else "failures" + toc.append( + f' ' + ) + sections.append(render_release_section(section_id, rdata, all_bug_candidates, zstream_info=zinfo)) + pr_section = render_pr_section(pr_data, all_bug_candidates, pr_status, pr_error) bugs_section = render_bugs_section(bugs_tab_data) if bugs_tab_data else "" images_section = render_images_section(images_tab_data) + zstream_js = "" + if zstream_checkbox: + zstream_js = """ +function filterZstream(on) { + var todayCheck = document.getElementById('filter-today'); + if (on && todayCheck && todayCheck.checked) { + todayCheck.checked = false; + filterToday(false); + } + document.querySelectorAll('#tab-periodics .toc > ul > li:not(.toc-zstream)').forEach(function(el) { el.style.display = on ? 'none' : ''; }); + document.querySelectorAll('#tab-periodics .toc > ul > li.toc-zstream').forEach(function(el) { el.style.display = on ? '' : 'none'; }); + document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section)').forEach(function(el) { el.style.display = on ? 'none' : ''; }); + document.querySelectorAll('#tab-periodics .zstream-section').forEach(function(el) { el.style.display = on ? '' : 'none'; }); +}""" + js_content = JS + zstream_js + return f"""\ @@ -1551,13 +1649,15 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p

Table of Contents

- +
+ +{zstream_checkbox} +
    {chr(10).join(toc)}
- {chr(10).join(sections)}
@@ -1576,7 +1676,7 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p

 

 

 

 

@@ -1756,15 +1856,28 @@ def main(): images_data = load_images_data(workdir, releases) images_tab_data = build_images_tab_data(images_data, releases) if images_data else None - # Set graphs directory for rendering - global _GRAPHS_DIR + # Set globals for rendering + global _GRAPHS_DIR, _COMPONENT + _COMPONENT = component graphs_dir = os.path.join(workdir, "graphs") if os.path.isdir(graphs_dir): _GRAPHS_DIR = graphs_dir + zstream_data = load_json(os.path.join(workdir, "zstream-summary.json")) + if zstream_data: + for zv, zinfo in zstream_data.items(): + if zinfo.get("status") == "skipped": + pr_title = zinfo.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + patch = m.group(1) if m else f"{zv}-zstream" + rdata = load_json(os.path.join(workdir, "jobs", f"release-{patch}-summary.json")) + if rdata is None: + rdata = {"total_failed": 0, "issues": [], "breakdown": _EMPTY_BREAKDOWN} + zinfo["_analyzed"] = rdata + # Generate HTML timestamp = datetime.now(timezone.utc) - html_content = generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error, bugs_tab_data, images_tab_data, index_data) + html_content = generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error, bugs_tab_data, images_tab_data, index_data, zstream_data) output_path = os.path.join(workdir, f"report-{component}-ci-doctor.html") with open(output_path, "w") as f: @@ -1812,6 +1925,17 @@ def main(): print(f" Release {release}: {repo_names}{grade_str}") else: print(" No container image data") + if zstream_data: + print(" Z-Stream:") + for zv in sorted(zstream_data, key=lambda v: [int(x) for x in v.split(".")]): + zi = zstream_data[zv] + zs = zi.get("status", "unknown") + extra = "" + if zs == "triggered": + extra = f' ({zi.get("jobs_triggered", 0)} jobs, PR {zi.get("pr", "")})' + elif zi.get("reason"): + extra = f' ({zi["reason"]})' + print(f" Release {zv}: {zs}{extra}") print(f"\nHTML report generated: {output_path}") diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index 569c7302..3450603a 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -121,6 +121,72 @@ cmd_prepare() { echo " Done: ${jobs_file}" >&2 done + # Collect and download for z-stream "skipped" releases + local zstream_summary="${WORKDIR}/zstream-summary.json" + declare -a ZSTREAM_RELEASES=() + local skipped_releases + skipped_releases=$(jq -r 'to_entries[] | select(.value.status == "skipped") | .key' "${zstream_summary}" 2>/dev/null || true) + + while IFS= read -r zrel; do + [[ -z "${zrel}" ]] && continue + # Extract patch version from PR title (e.g. "chore: Release 4.19.3" → "4.19.3") + local patch_version + patch_version=$(jq -r --arg r "${zrel}" '.[$r].pr_title // ""' "${zstream_summary}" 2>/dev/null \ + | grep -oP '\d+\.\d+\.\d+' || echo "${zrel}-zstream") + + # Skip if already collected as a regular release + if [[ -f "${WORKDIR}/jobs/release-${patch_version}-jobs.json" ]] && \ + [[ "$(jq 'length' "${WORKDIR}/jobs/release-${patch_version}-jobs.json")" -gt 0 ]]; then + echo " Skipping z-stream ${patch_version}: already collected" >&2 + continue + fi + + echo "=== Z-Stream ${zrel} → ${patch_version} (skipped — same digest) ===" >&2 + local zjobs_file="${WORKDIR}/jobs/release-${patch_version}-jobs.json" + + echo " Collecting failed periodic jobs..." >&2 + local zraw_json zraw_err + zraw_err=$(mktemp) + if ! zraw_json=$(bash "${SCRIPT_DIR}/prow-jobs-for-release.sh" "${COMPONENT}" "${zrel}" 2>"${zraw_err}"); then + echo " ERROR: failed to collect z-stream jobs for ${zrel}:" >&2 + local zerr_msg + zerr_msg=$(cat "${zraw_err}") + echo "${zerr_msg}" >&2 + rm -f "${zraw_err}" + echo "[]" > "${zjobs_file}" + release_errors["${patch_version}"]="${zerr_msg:-data collection failed}" + echo "${release_errors["${patch_version}"]}" > "${WORKDIR}/jobs/release-${patch_version}-error.txt" + continue + fi + rm -f "${zraw_err}" + + local zfiltered_json + zfiltered_json=$(echo "${zraw_json}" | jq '[.[] | select(.type == "periodic")]') + + local zcount + zcount=$(echo "${zfiltered_json}" | jq 'length') + + if [[ "${zcount}" -eq 0 ]]; then + echo " No failed periodic jobs found" >&2 + echo "[]" > "${zjobs_file}" + ZSTREAM_RELEASES+=("${patch_version}") + continue + fi + + echo " Found ${zcount} failed periodic jobs, downloading artifacts..." >&2 + local zdl_err + zdl_err=$(mktemp) + echo "${zfiltered_json}" | \ + bash "${SCRIPT_DIR}/download-jobs.sh" --workdir "${WORKDIR}" 2>"${zdl_err}" \ + > "${zjobs_file}" + [[ -s "${zdl_err}" ]] && cat "${zdl_err}" >&2 + rm -f "${zdl_err}" + + total_jobs=$((total_jobs + zcount)) + ZSTREAM_RELEASES+=("${patch_version}") + echo " Done: ${zjobs_file}" >&2 + done <<< "${skipped_releases}" + # Collect and download for rebase PRs if ${do_rebase}; then echo "=== Rebase Pull Requests ===" >&2 @@ -266,6 +332,22 @@ cmd_prepare() { result=$(jq -n --arg w "${WORKDIR}" --argjson rel "${releases_json}" \ '{workdir: $w, releases: $rel}') + # Add z-stream releases to output (uses patch versions, same release-* naming) + if [[ ${#ZSTREAM_RELEASES[@]} -gt 0 ]]; then + local zstream_json="[]" + for zpatch in "${ZSTREAM_RELEASES[@]}"; do + local zjobs_file="${WORKDIR}/jobs/release-${zpatch}-jobs.json" + local zcount=0 + if [[ -f "${zjobs_file}" ]]; then + zcount=$(jq 'length' "${zjobs_file}") + fi + zstream_json=$(echo "${zstream_json}" | jq \ + --arg r "${zpatch}" --argjson c "${zcount}" --arg f "${zjobs_file}" \ + '. + [{release: $r, jobs: $c, jobs_file: $f}]') + done + result=$(echo "${result}" | jq --argjson z "${zstream_json}" '. + {zstream: $z}') + fi + if ${do_rebase}; then local prs_file="${WORKDIR}/jobs/prs-jobs.json" local pr_job_count=0 @@ -329,14 +411,22 @@ cmd_finalize() { IFS=',' read -ra RELEASES <<< "${releases_arg}" - # Aggregate each release - for release in "${RELEASES[@]}"; do - release=$(echo "${release}" | xargs) - echo "=== Aggregating release ${release} ===" >&2 - python3 "${SCRIPT_DIR}/aggregate.py" \ - --release "${release}" --workdir "${WORKDIR}" >/dev/null || \ - echo " WARNING: aggregation failed for ${release}" >&2 - done + # Aggregate all job files (regular releases + z-stream patch versions) + local all_job_files + all_job_files=$(find "${WORKDIR}/jobs" -name 'release-*-jobs.json' 2>/dev/null | sort || true) + while IFS= read -r jobs_file; do + [[ -z "${jobs_file}" ]] && continue + local release + release=$(basename "${jobs_file}" | sed 's/^release-//; s/-jobs\.json$//') + local reports + reports=$(find "${WORKDIR}/jobs" -name "release-${release}-job-*.txt" 2>/dev/null | head -1) + if [[ -n "${reports}" ]]; then + echo "=== Aggregating release ${release} ===" >&2 + python3 "${SCRIPT_DIR}/aggregate.py" \ + --release "${release}" --workdir "${WORKDIR}" >/dev/null || \ + echo " WARNING: aggregation failed for ${release}" >&2 + fi + done <<< "${all_job_files}" # Aggregate PRs (if job files exist) local pr_files @@ -356,7 +446,8 @@ cmd_finalize() { local images_dir="${WORKDIR}/images" mkdir -p "${images_dir}" - for repo in ${repos}; do + while IFS= read -r repo; do + [[ -z "${repo}" ]] && continue local repo_slug="${repo//\//@}" # Fetch catalog repository ID (needed for catalog URLs) @@ -383,7 +474,7 @@ cmd_finalize() { fi done echo "${tmp_merged}" > "${outfile}" - done + done <<< "${repos}" fi # Extract index image info (LVMS-specific, no-op for other components) From 28592128d0b8e3f7e6530ea31efe4c7f8a7395bd Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:02:51 +0530 Subject: [PATCH 2/8] Fix review findings in z-stream trigger script - Switch from -on-push tags (which expire) to v{version}-{commit} merge tags - Fix empty triggered_jobs producing [""] instead of [] Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 45e1f71f..66084100 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -221,17 +221,22 @@ resolve_snapshot_to_digest() { snap_sec=${time_str:4:2} snap_epoch=$(date -d "${snap_year}-${snap_month}-${snap_day}T${snap_hour}:${snap_min}:${snap_sec}Z" +%s 2>/dev/null || echo 0) - # Query quay for on-push build-image-index tags matching this version + # Convert version prefix (lvm-operator-catalog-4-16) to v4.16 for tag matching + local version_dot + version_dot=$(echo "${version_prefix}" | sed 's/lvm-operator-catalog-//; s/-/./') + + # Query quay for v{version} commit tags (not -on-push, which expire) local tags_json tags_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ - "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:${version_prefix}-on-push") + "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:v${version_dot}-") - # Find build-image-index tags created after the snapshot timestamp - # Pick the closest one (smallest positive time delta) + # Select only raw commit tags (v4.16-{40-hex}), exclude auxiliary suffixes + # Pick the closest one after the snapshot timestamp (smallest positive delta) local result - result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" ' + result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" --arg vpfx "v${version_dot}-" ' [.tags[] - | select(.name | endswith("build-image-index")) + | select(.name | startswith($vpfx)) + | select(.name | test("^v[0-9]+\\.[0-9]+-[a-f0-9]{40}$")) | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) | .delta = (.tag_epoch - ($snap_epoch | tonumber)) | select(.delta >= 0 and .delta < 1800) @@ -441,7 +446,7 @@ process_release() { --arg snap "${snapshot}" \ --argjson triggered "${triggered}" \ --argjson failed "${failed}" \ - --argjson jobs "$(printf '%s\n' "${triggered_jobs[@]}" | jq -R . | jq -s .)" \ + --argjson jobs "$(if [[ ${#triggered_jobs[@]} -gt 0 ]]; then printf '%s\n' "${triggered_jobs[@]}" | jq -R . | jq -s .; else echo '[]'; fi)" \ '{($r): {status: "triggered", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs_triggered: $triggered, jobs_failed: $failed, jobs: $jobs}}' \ > "${WORKDIR}/zstream-${release}.json" } From f8795e0e9a1ad95733ec04a31a10c1fb9b8cde45 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:08:23 +0530 Subject: [PATCH 3/8] Sort fetch_last_runs results by startTime to pick latest run Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 66084100..e9c5a4c6 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -316,6 +316,7 @@ fetch_last_runs() { entry=$(echo "${prow_json}" | sed 's/^var allBuilds = //' | \ jq -r --arg n "${test_name}" ' [.items[] | select(.status.state == "success" or .status.state == "failure" or .status.state == "error" or .status.state == "aborted")] | + sort_by(.status.startTime) | reverse | if length > 0 then .[0] else null end | if . then {name: $n, state: .status.state, url: .status.url, started: .status.startTime} else {name: $n, state: "unknown"} end ' 2>/dev/null || echo "{\"name\": \"${test_name}\", \"state\": \"unknown\"}") From 41d15a6c0a4f8a0b42ebd008a5a5e47169730bab Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:47:52 +0530 Subject: [PATCH 4/8] Use closest tag by absolute time delta for snapshot resolution The v{version}-{commit} merge tags can appear before or after the snapshot timestamp, so pick the closest match regardless of direction instead of requiring a narrow 30-minute window. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index e9c5a4c6..f79544ea 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -231,16 +231,15 @@ resolve_snapshot_to_digest() { "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:v${version_dot}-") # Select only raw commit tags (v4.16-{40-hex}), exclude auxiliary suffixes - # Pick the closest one after the snapshot timestamp (smallest positive delta) + # Pick the tag closest to the snapshot timestamp (smallest absolute delta) local result result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" --arg vpfx "v${version_dot}-" ' [.tags[] | select(.name | startswith($vpfx)) | select(.name | test("^v[0-9]+\\.[0-9]+-[a-f0-9]{40}$")) | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) - | .delta = (.tag_epoch - ($snap_epoch | tonumber)) - | select(.delta >= 0 and .delta < 1800) - ] | sort_by(.delta) | .[0] // empty | + | .abs_delta = ((.tag_epoch - ($snap_epoch | tonumber)) | fabs) + ] | sort_by(.abs_delta) | .[0] // empty | [.name, .manifest_digest] | @tsv ') From 0aada91f3c37fda02862e38140ab48b5f2c4487d Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:54:32 +0530 Subject: [PATCH 5/8] Add retry with backoff for Gangway 429 rate limiting Retry up to 3 times on HTTP 429 with exponential backoff (15s, 30s). Increase base delay between triggers from 5s to 30s. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 38 ++++++++++++++-------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index f79544ea..20d7666d 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -274,21 +274,31 @@ trigger_job() { return 0 fi - local http_code - http_code=$(curl -sSL -X POST -o /dev/stderr -w '%{http_code}' \ - --connect-timeout 10 --max-time 30 \ - -H "Authorization: Bearer ${MY_APPCI_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "${body}" \ - "${GANGWAY_API}/v1/executions/${job_name}" 2>/dev/null) - - if [[ "${http_code}" == "200" ]]; then - info " triggered: ${job_name}" - return 0 - else + local http_code attempt max_retries=3 delay=15 + + for ((attempt = 1; attempt <= max_retries; attempt++)); do + http_code=$(curl -sSL -X POST -o /dev/stderr -w '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${MY_APPCI_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${body}" \ + "${GANGWAY_API}/v1/executions/${job_name}" 2>/dev/null) + + if [[ "${http_code}" == "200" ]]; then + info " triggered: ${job_name}" + return 0 + fi + + if [[ "${http_code}" == "429" && ${attempt} -lt ${max_retries} ]]; then + warn " rate limited (attempt ${attempt}/${max_retries}), retrying in ${delay}s: ${job_name}" + sleep "${delay}" + delay=$((delay * 2)) + continue + fi + warn " failed (HTTP ${http_code}): ${job_name}" return 1 - fi + done } # --------------------------------------------------------------------------- @@ -427,7 +437,7 @@ process_release() { else failed=$((failed + 1)) fi - ${DRY_RUN} || sleep 5 + ${DRY_RUN} || sleep 30 done info "${release}: ${triggered} triggered, ${failed} failed" From 7dfc019dd24e3e79c317260e3cfc13727f207d9f Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Fri, 3 Jul 2026 12:57:09 +0530 Subject: [PATCH 6/8] lvms-zstream: filter PRs by release-management branch and match by snapshot content Filter GitHub PR search to base=release-management branch (all LVMS release PRs target this branch), eliminating the need for title regex matching. Instead, match PRs by checking if their catalog files contain a snapshot referencing the target version. This handles varying title formats and reduces API calls (5 PRs vs 100). The snapshot is now extracted during PR discovery and returned alongside the PR number/title, removing the need for a separate extract_snapshot call. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 66 +++++++++------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 20d7666d..b7eca337 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -147,15 +147,32 @@ find_zstream_pr() { local major minor major=$(echo "${release}" | cut -d. -f1) minor=$(echo "${release}" | cut -d. -f2) + local catalog_prefix="lvm-operator-catalog-${major}-${minor}" local prs_json - prs_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls?state=open&per_page=100") - - echo "${prs_json}" | jq -r --arg maj "${major}" --arg min "${minor}" ' - [.[] | select(.title | test(": Release " + $maj + "\\." + $min + "\\."; "i"))] | - if length > 0 then .[0] else empty end | - [.number, .title] | @tsv - ' + prs_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls?state=open&base=release-management&per_page=100") + + local pr_count + pr_count=$(echo "${prs_json}" | jq 'length') + + local i + for ((i = 0; i < pr_count; i++)); do + local pr_number pr_title + pr_number=$(echo "${prs_json}" | jq -r ".[$i].number") + pr_title=$(echo "${prs_json}" | jq -r ".[$i].title") + + local files_json snapshot + files_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}/files") + snapshot=$(echo "${files_json}" | jq -r ' + [.[] | select(.filename | contains("catalog")) | .patch // ""] | + join("\n") + ' | { grep -oP "(?<=snapshot: )${catalog_prefix}\\S*" || true; } | head -1) + + if [[ -n "${snapshot}" ]]; then + printf '%s\t%s\t%s\n' "${pr_number}" "${pr_title}" "${snapshot}" + return 0 + fi + done } # --------------------------------------------------------------------------- @@ -176,22 +193,6 @@ check_pr_merged() { fi } -# --------------------------------------------------------------------------- -# GitHub: extract snapshot name from PR diff -# --------------------------------------------------------------------------- - -extract_snapshot() { - local pr_number="$1" - - local files_json - files_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}/files") - - echo "${files_json}" | jq -r ' - [.[] | select(.filename | contains("catalog")) | .patch // ""] | - join("\n") - ' | { grep -oP '(?<=snapshot: )\S+' || true; } | head -1 -} - # --------------------------------------------------------------------------- # Quay: resolve snapshot to digest # --------------------------------------------------------------------------- @@ -342,9 +343,6 @@ fetch_last_runs() { process_release() { local release="$1" - local major minor - major=$(echo "${release}" | cut -d. -f1) - minor=$(echo "${release}" | cut -d. -f2) info "--- Release ${release} ---" @@ -375,23 +373,11 @@ process_release() { return 0 fi - local pr_number pr_title + local pr_number pr_title snapshot pr_number=$(echo "${pr_info}" | cut -f1) pr_title=$(echo "${pr_info}" | cut -f2) + snapshot=$(echo "${pr_info}" | cut -f3) info "Found PR #${pr_number}: ${pr_title}" - - # Extract snapshot from PR - local snapshot - snapshot=$(extract_snapshot "${pr_number}") - - if [[ -z "${snapshot}" ]]; then - warn "Could not extract catalog snapshot from PR #${pr_number}" - jq -n --arg r "${release}" --arg pr "${pr_number}" \ - '{($r): {status: "error", reason: "no snapshot in PR", pr: ("#" + $pr)}}' \ - > "${WORKDIR}/zstream-${release}.json" - return 0 - fi - info "Snapshot: ${snapshot}" # Resolve to digest From 7bb953ff33d70ebc37fddcb3a64461a108dc3da9 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Fri, 3 Jul 2026 13:33:16 +0530 Subject: [PATCH 7/8] lvms-zstream: guard against null last_modified in quay tag resolution Skip tags with missing last_modified before calling strptime to prevent jq pipeline abort during digest resolution. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index b7eca337..4e1ecb8a 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -238,6 +238,7 @@ resolve_snapshot_to_digest() { [.tags[] | select(.name | startswith($vpfx)) | select(.name | test("^v[0-9]+\\.[0-9]+-[a-f0-9]{40}$")) + | select(.last_modified != null and .last_modified != "") | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) | .abs_delta = ((.tag_epoch - ($snap_epoch | tonumber)) | fabs) ] | sort_by(.abs_delta) | .[0] // empty | From aef588d5fb10bfd3e2b6d1315ea29c9ddc95cbf4 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Fri, 3 Jul 2026 23:50:06 +0530 Subject: [PATCH 8/8] lvms-zstream: separate z-stream collection from doctor.sh Move z-stream artifact collection out of the shared doctor.sh into zstream-trigger.sh so doctor stays reporting-only with no LVMS-specific logic. For skipped releases (same digest), the trigger script now collects failed periodic jobs and downloads artifacts into the workdir using the same release-*-jobs.json naming, so doctor's finalize discovers them automatically. Also make --releases a required argument instead of hardcoding the release list. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 74 ++++++++++++++++++- plugins/lvms-ci/skills/doctor/SKILL.md | 16 ++--- plugins/shared/scripts/doctor.sh | 84 +--------------------- 3 files changed, 80 insertions(+), 94 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 4e1ecb8a..f0196a02 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -12,19 +12,22 @@ set -euo pipefail # the last run (tracked via a state file). # # Usage: -# zstream-trigger.sh [--releases 4.14,4.16,...] [--dry-run] [--workdir /path] +# zstream-trigger.sh --releases 4.14,4.16,... [--dry-run] [--workdir /path] # [--force] [--token-file /path/to/token] # # Environment: # GITHUB_TOKEN Optional. GitHub PAT for higher API rate limits. # MY_APPCI_TOKEN Required (unless --dry-run). Gangway API bearer token. -RELEASES="4.14,4.16,4.18,4.19,4.20,4.21" +RELEASES="" DRY_RUN=false FORCE=false WORKDIR="" TOKEN_FILE="" +TRIGGER_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +COMPONENT="lvm-operator" + GANGWAY_API="https://gangway-ci.apps.ci.l2s4.p1.openshiftapps.com" GITHUB_API="https://api.github.com" QUAY_API="https://quay.io/api/v1" @@ -63,7 +66,7 @@ usage() { Usage: $(basename "$0") [OPTIONS] Options: - --releases LIST Comma-separated releases (default: ${RELEASES}) + --releases LIST Comma-separated releases (required) --dry-run Print what would be triggered without calling Gangway --force Trigger even if digest hasn't changed since last run --workdir DIR Write summary JSON here (default: /tmp/lvms-zstream.) @@ -92,6 +95,11 @@ while [[ ${#} -gt 0 ]]; do esac done +if [[ -z "${RELEASES}" ]]; then + err "--releases is required" + usage +fi + WORKDIR="${WORKDIR:-/tmp/lvms-zstream.$(date +%y%m%d)}" mkdir -p "${WORKDIR}" @@ -338,6 +346,64 @@ fetch_last_runs() { echo "${results}" } +# --------------------------------------------------------------------------- +# Collect artifacts for a skipped release (same digest, already tested) +# --------------------------------------------------------------------------- + +collect_skipped_artifacts() { + local release="$1" + local pr_title="$2" + + local patch_version + patch_version=$(echo "${pr_title}" | grep -oP '\d+\.\d+\.\d+' || echo "${release}-zstream") + + local jobs_dir="${WORKDIR}/jobs" + mkdir -p "${jobs_dir}" + + local jobs_file="${jobs_dir}/release-${patch_version}-jobs.json" + + # Skip if already collected + if [[ -f "${jobs_file}" ]] && [[ "$(jq 'length' "${jobs_file}")" -gt 0 ]]; then + info " Artifacts already collected for ${patch_version}" + return 0 + fi + + info " Collecting failed periodic jobs for ${release}..." + local raw_json raw_err + raw_err=$(mktemp) + if ! raw_json=$(bash "${TRIGGER_DIR}/prow-jobs-for-release.sh" "${COMPONENT}" "${release}" 2>"${raw_err}"); then + warn "Failed to collect jobs for ${release}:" + cat "${raw_err}" >&2 + rm -f "${raw_err}" + echo "[]" > "${jobs_file}" + return 0 + fi + rm -f "${raw_err}" + + local filtered_json + filtered_json=$(echo "${raw_json}" | jq '[.[] | select(.type == "periodic")]') + + local count + count=$(echo "${filtered_json}" | jq 'length') + + if [[ "${count}" -eq 0 ]]; then + info " No failed periodic jobs found for ${patch_version}" + echo "[]" > "${jobs_file}" + return 0 + fi + + info " Found ${count} failed periodic jobs, downloading artifacts..." + local dl_err + dl_err=$(mktemp) + echo "${filtered_json}" | \ + bash "${TRIGGER_DIR}/download-jobs.sh" --workdir "${WORKDIR}" 2>"${dl_err}" \ + > "${jobs_file}" + [[ -s "${dl_err}" ]] && cat "${dl_err}" >&2 + rm -f "${dl_err}" + + info " Done: ${jobs_file}" +} + # --------------------------------------------------------------------------- # Process a single release # --------------------------------------------------------------------------- @@ -407,6 +473,8 @@ process_release() { --argjson jobs "${last_runs}" \ '{($r): {status: "skipped", reason: "same digest", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs: $jobs}}' \ > "${WORKDIR}/zstream-${release}.json" + + collect_skipped_artifacts "${release}" "${pr_title}" return 0 fi fi diff --git a/plugins/lvms-ci/skills/doctor/SKILL.md b/plugins/lvms-ci/skills/doctor/SKILL.md index 80ab000e..e68fc8b1 100644 --- a/plugins/lvms-ci/skills/doctor/SKILL.md +++ b/plugins/lvms-ci/skills/doctor/SKILL.md @@ -82,7 +82,7 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa Use the Write tool to save the file. The file must contain the complete analysis report." ``` -3. If the prepare output contains a `"zstream"` key, process those entries the same way — they use patch versions (e.g., `4.19.3`) as the `` value but follow the identical naming and analysis pattern. +3. If z-stream job files exist in `/jobs/` (e.g., `release-4.19.3-jobs.json` — written by `zstream-trigger.sh`), analyze those jobs the same way — they use patch versions (e.g., `4.19.3`) as the `` value but follow the identical naming and analysis pattern. 4. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. 5. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. @@ -147,20 +147,20 @@ HTML report generated: /report-lvm-operator-ci-doctor.html ### Z-Stream Integration (Optional) -If a `zstream-summary.json` file exists in the workdir and contains releases with `"status": "skipped"` (same digest already tested), the HTML report will include a **Z-Stream** checkbox on the Periodics tab. When checked, it switches the view to show z-stream release status per version (PR info, image digest, triggered jobs). +Run `zstream-trigger.sh` before doctor to populate z-stream data. For skipped releases (same digest already tested), the trigger script collects failed periodic job artifacts into `/jobs/release--jobs.json` — these are discovered automatically by the finalize step alongside regular releases. -To populate z-stream data before generating the report: +The HTML report includes a **Z-Stream** checkbox on the Periodics tab when `zstream-summary.json` exists in the workdir. + +To populate z-stream data (dry-run — no job triggering): ```text -bash plugins/lvms-ci/scripts/zstream-trigger.sh --dry-run --workdir +bash plugins/lvms-ci/scripts/zstream-trigger.sh --releases --dry-run --workdir ``` -This queries GitHub for open z-stream release PRs on `openshift/lvm-operator`, resolves catalog image digests via quay.io, and writes `zstream-summary.json` to the workdir — without triggering any jobs (dry-run mode). - -To actually trigger z-stream e2e jobs (requires `MY_APPCI_TOKEN`): +To trigger z-stream e2e jobs and collect skipped release artifacts (requires `MY_APPCI_TOKEN`): ```text -bash plugins/lvms-ci/scripts/zstream-trigger.sh --workdir +bash plugins/lvms-ci/scripts/zstream-trigger.sh --releases --workdir ``` ## Related Skills diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index 3450603a..f87c768d 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -121,72 +121,6 @@ cmd_prepare() { echo " Done: ${jobs_file}" >&2 done - # Collect and download for z-stream "skipped" releases - local zstream_summary="${WORKDIR}/zstream-summary.json" - declare -a ZSTREAM_RELEASES=() - local skipped_releases - skipped_releases=$(jq -r 'to_entries[] | select(.value.status == "skipped") | .key' "${zstream_summary}" 2>/dev/null || true) - - while IFS= read -r zrel; do - [[ -z "${zrel}" ]] && continue - # Extract patch version from PR title (e.g. "chore: Release 4.19.3" → "4.19.3") - local patch_version - patch_version=$(jq -r --arg r "${zrel}" '.[$r].pr_title // ""' "${zstream_summary}" 2>/dev/null \ - | grep -oP '\d+\.\d+\.\d+' || echo "${zrel}-zstream") - - # Skip if already collected as a regular release - if [[ -f "${WORKDIR}/jobs/release-${patch_version}-jobs.json" ]] && \ - [[ "$(jq 'length' "${WORKDIR}/jobs/release-${patch_version}-jobs.json")" -gt 0 ]]; then - echo " Skipping z-stream ${patch_version}: already collected" >&2 - continue - fi - - echo "=== Z-Stream ${zrel} → ${patch_version} (skipped — same digest) ===" >&2 - local zjobs_file="${WORKDIR}/jobs/release-${patch_version}-jobs.json" - - echo " Collecting failed periodic jobs..." >&2 - local zraw_json zraw_err - zraw_err=$(mktemp) - if ! zraw_json=$(bash "${SCRIPT_DIR}/prow-jobs-for-release.sh" "${COMPONENT}" "${zrel}" 2>"${zraw_err}"); then - echo " ERROR: failed to collect z-stream jobs for ${zrel}:" >&2 - local zerr_msg - zerr_msg=$(cat "${zraw_err}") - echo "${zerr_msg}" >&2 - rm -f "${zraw_err}" - echo "[]" > "${zjobs_file}" - release_errors["${patch_version}"]="${zerr_msg:-data collection failed}" - echo "${release_errors["${patch_version}"]}" > "${WORKDIR}/jobs/release-${patch_version}-error.txt" - continue - fi - rm -f "${zraw_err}" - - local zfiltered_json - zfiltered_json=$(echo "${zraw_json}" | jq '[.[] | select(.type == "periodic")]') - - local zcount - zcount=$(echo "${zfiltered_json}" | jq 'length') - - if [[ "${zcount}" -eq 0 ]]; then - echo " No failed periodic jobs found" >&2 - echo "[]" > "${zjobs_file}" - ZSTREAM_RELEASES+=("${patch_version}") - continue - fi - - echo " Found ${zcount} failed periodic jobs, downloading artifacts..." >&2 - local zdl_err - zdl_err=$(mktemp) - echo "${zfiltered_json}" | \ - bash "${SCRIPT_DIR}/download-jobs.sh" --workdir "${WORKDIR}" 2>"${zdl_err}" \ - > "${zjobs_file}" - [[ -s "${zdl_err}" ]] && cat "${zdl_err}" >&2 - rm -f "${zdl_err}" - - total_jobs=$((total_jobs + zcount)) - ZSTREAM_RELEASES+=("${patch_version}") - echo " Done: ${zjobs_file}" >&2 - done <<< "${skipped_releases}" - # Collect and download for rebase PRs if ${do_rebase}; then echo "=== Rebase Pull Requests ===" >&2 @@ -332,22 +266,6 @@ cmd_prepare() { result=$(jq -n --arg w "${WORKDIR}" --argjson rel "${releases_json}" \ '{workdir: $w, releases: $rel}') - # Add z-stream releases to output (uses patch versions, same release-* naming) - if [[ ${#ZSTREAM_RELEASES[@]} -gt 0 ]]; then - local zstream_json="[]" - for zpatch in "${ZSTREAM_RELEASES[@]}"; do - local zjobs_file="${WORKDIR}/jobs/release-${zpatch}-jobs.json" - local zcount=0 - if [[ -f "${zjobs_file}" ]]; then - zcount=$(jq 'length' "${zjobs_file}") - fi - zstream_json=$(echo "${zstream_json}" | jq \ - --arg r "${zpatch}" --argjson c "${zcount}" --arg f "${zjobs_file}" \ - '. + [{release: $r, jobs: $c, jobs_file: $f}]') - done - result=$(echo "${result}" | jq --argjson z "${zstream_json}" '. + {zstream: $z}') - fi - if ${do_rebase}; then local prs_file="${WORKDIR}/jobs/prs-jobs.json" local pr_job_count=0 @@ -411,7 +329,7 @@ cmd_finalize() { IFS=',' read -ra RELEASES <<< "${releases_arg}" - # Aggregate all job files (regular releases + z-stream patch versions) + # Aggregate all job files local all_job_files all_job_files=$(find "${WORKDIR}/jobs" -name 'release-*-jobs.json' 2>/dev/null | sort || true) while IFS= read -r jobs_file; do