From 8783a207f879d37c70c90e6db1e0873797b13c6c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jun 2026 18:21:49 +0200 Subject: [PATCH 01/23] [Backport pipeline] Restrict UI access to ecosystem team members Gate the integrations-backport pipeline so that UI-triggered builds are only allowed for members of the `ecosystem` team, using BUILDKITE_BUILD_CREATOR_TEAMS. Trigger-job builds from the integrations pipeline remain unrestricted. Co-authored-by: Claude Sonnet 4.6 --- .buildkite/pipeline.backport.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 0f37b461cac..afff22dfe83 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -13,11 +13,11 @@ steps: - label: "Check that it runs from UI" key: "check-ui" command: - - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG is used only for running from UI or a trigger step!\" --style 'warning'" + - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG pipeline can only be triggered from the UI by members of the 'ecosystem' team, or via a trigger step from the 'integrations' pipeline.\" --style 'warning'" - "exit 1" if: | !( - build.source == 'ui' || + (build.source == 'ui' && build.env('BUILDKITE_BUILD_CREATOR_TEAMS') =~ /(^|:)ecosystem(:|$)/) || (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') ) From 27f694b8aa793b504b7f5e181334e29afe28a077 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jun 2026 12:08:42 +0200 Subject: [PATCH 02/23] [Backport pipeline] Add PR-driven backport branch creation step Add trigger-backport-create step to pipeline.yml and its companion script trigger_backport_create.sh. On every push to main that touches .backports.yml, the script detects new entries (vs HEAD^), resolves the merged PR number via the GitHub API, and uploads trigger steps that fire the integrations-backport pipeline with DRY_RUN=false, passing PR_NUMBER so the triggered build can post a result comment. Co-authored-by: Claude Sonnet 4.6 --- .buildkite/pipeline.yml | 14 ++ .buildkite/scripts/trigger_backport_create.sh | 137 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 .buildkite/scripts/trigger_backport_create.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 01bcc6d2967..cd4e30e2472 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -99,6 +99,20 @@ steps: build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" && build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == "main" + - label: ":git: Create backport branches for new entries" + key: "trigger-backport-create" + command: ".buildkite/scripts/trigger_backport_create.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + if_changed: + - ".backports.yml" + if: | + build.env('BUILDKITE_PULL_REQUEST') == "false" && + build.env('BUILDKITE_BRANCH') == "main" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" + - label: ":junit: Sources Junit annotate" agents: # requires at least "bash", "curl" and "git" diff --git a/.buildkite/scripts/trigger_backport_create.sh b/.buildkite/scripts/trigger_backport_create.sh new file mode 100644 index 00000000000..fe11b6b8ad8 --- /dev/null +++ b/.buildkite/scripts/trigger_backport_create.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Triggered from the main pipeline when .backports.yml changes on a push to main. +# For each entry that is new (absent from HEAD^), uploads a trigger step that +# runs the integrations-backport pipeline with DRY_RUN=false. + +source .buildkite/scripts/common.sh + +set -euo pipefail + +if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then + echo "Pull request build, skipping backport create trigger" + exit 0 +fi + +if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then + echo "Not on main branch (branch: ${BUILDKITE_BRANCH}), skipping backport create trigger" + exit 0 +fi + +add_bin_path +with_yq +with_mage +with_github_cli + +if ! git diff --name-only HEAD^ HEAD | grep -qE '^\.backports\.yml$'; then + echo ".backports.yml not changed, skipping backport create trigger" + exit 0 +fi + +echo "--- .backports.yml changed — finding new entries" + +OLD_INVENTORY="" +PIPELINE_FILE="" + +cleanup() { + local exit_code=$? + [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" + [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" + exit "${exit_code}" +} +trap cleanup EXIT + +PR_NUMBER=$(gh api "repos/elastic/integrations/commits/${BUILDKITE_COMMIT}/pulls" --jq '.[0].number' 2>/dev/null || true) +if [[ -n "${PR_NUMBER}" ]]; then + echo "Associated PR: #${PR_NUMBER}" +else + echo "Could not resolve PR number for commit ${BUILDKITE_COMMIT}, PR comments will be skipped" +fi + +OLD_INVENTORY="$(mktemp)" +NEW_INVENTORY=".backports.yml" + +if ! git show "HEAD^:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then + echo ".backports.yml is new on main — skipping create for initial entries" + echo "To create branches for these entries, trigger the integrations-backport pipeline manually." + exit 0 +fi + +if ! yq -e '.backports' "${OLD_INVENTORY}" > /dev/null; then + echo "ERROR: old inventory is not valid YAML or missing 'backports' key: ${OLD_INVENTORY}" + exit 1 +fi + +if ! yq -e '.backports' "${NEW_INVENTORY}" > /dev/null; then + echo "ERROR: new inventory is not valid YAML or missing 'backports' key: ${NEW_INVENTORY}" + exit 1 +fi + +PIPELINE_FILE="$(mktemp --suffix=.yml)" +entries_found=0 + +while IFS= read -r branch; do + entry=".backports[] | select(.branch == \"${branch}\")" + + active_exit=0 + mage CheckBackportBranchActive "${branch}" || active_exit=$? + if [[ "${active_exit}" -eq 2 ]]; then + echo "ERROR: failed to check active status for branch '${branch}'" + exit 1 + fi + if [[ "${active_exit}" -ne 0 ]]; then + echo " Skipping inactive entry: ${branch}" + continue + fi + + # Only trigger for entries that are new (absent from the pre-merge state). + # If the entry already existed, the branch is already created; nothing to provision. + old_branch="$(yq "${entry} | .branch" "${OLD_INVENTORY}")" + + if [[ -n "${old_branch}" ]]; then + echo " Skipping existing entry: ${branch} (already present before merge)" + continue + fi + + pkg="$(yq "${entry} | .package" "${NEW_INVENTORY}")" + base_version="$(yq "${entry} | .base_version" "${NEW_INVENTORY}")" + base_commit="$(yq "${entry} | .base_commit" "${NEW_INVENTORY}")" + + echo " Queuing create: ${branch} (package=${pkg} version=${base_version} base_commit=${base_commit})" + + if [[ "${entries_found}" -eq 0 ]]; then + printf 'steps:\n' > "${PIPELINE_FILE}" + fi + + cat >> "${PIPELINE_FILE}" <> "${PIPELINE_FILE}" < Date: Wed, 17 Jun 2026 12:28:21 +0200 Subject: [PATCH 03/23] [Backport pipeline] Add PR notification step and script Add notify_backport_pr.sh to post a success or failure comment on the merged PR after backport branch creation. Add the notify-pr step to pipeline.backport.yml, gated on PR_NUMBER being set, using buildkite-agent step get "outcome" to distinguish pass from failure. Also add GH_CLI_VERSION to the backport pipeline env. Co-authored-by: Claude Sonnet 4.6 --- .buildkite/pipeline.backport.yml | 19 +++++++++++ .buildkite/scripts/backport_branch.sh | 2 +- .buildkite/scripts/notify_backport_pr.sh | 42 ++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 .buildkite/scripts/notify_backport_pr.sh diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index afff22dfe83..27038bff122 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -5,6 +5,7 @@ name: "integrations-backport" env: SETUP_GVM_VERSION: "v0.6.0" YQ_VERSION: 'v4.35.2' + GH_CLI_VERSION: "2.29.0" # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" @@ -70,3 +71,21 @@ steps: depends_on: - step: "input-variables" allow_failure: false + + - label: ":github: Notify PR" + key: "notify-pr" + command: | + outcome=$$(buildkite-agent step get "outcome" --step "create-backport-branch") + if [[ "$${outcome}" == "passed" ]]; then + NOTIFY_STATUS=success .buildkite/scripts/notify_backport_pr.sh + else + NOTIFY_STATUS=failure .buildkite/scripts/notify_backport_pr.sh + fi + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "create-backport-branch" + allow_failure: true + if: "build.env('PR_NUMBER') != ''" diff --git a/.buildkite/scripts/backport_branch.sh b/.buildkite/scripts/backport_branch.sh index 3248c90f093..6fe245c96e5 100755 --- a/.buildkite/scripts/backport_branch.sh +++ b/.buildkite/scripts/backport_branch.sh @@ -342,7 +342,7 @@ fi echo "--- Creating the branch: $BACKPORT_BRANCH_NAME from the commit: $BASE_COMMIT" createLocalBackportBranch "$BACKPORT_BRANCH_NAME" "$BASE_COMMIT" -MSG="The backport branch: **$BACKPORT_BRANCH_NAME** has been created." +MSG="The backport branch: **$BACKPORT_BRANCH_NAME** has been created" echo "+++ Adding CI files into the branch ${BACKPORT_BRANCH_NAME}" updateBackportBranchContents diff --git a/.buildkite/scripts/notify_backport_pr.sh b/.buildkite/scripts/notify_backport_pr.sh new file mode 100644 index 00000000000..381b0613002 --- /dev/null +++ b/.buildkite/scripts/notify_backport_pr.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Posts a comment on the PR that triggered the backport branch creation, +# reporting success or failure of the branch creation step. +# Expects env vars: PR_NUMBER, NOTIFY_STATUS (success|failure), +# BACKPORT_BRANCH_NAME, PACKAGE_NAME, PACKAGE_VERSION, +# BUILDKITE_BUILD_URL + +source .buildkite/scripts/common.sh + +set -euo pipefail + +if [[ -z "${PR_NUMBER:-}" ]]; then + echo "PR_NUMBER not set, skipping PR notification" + exit 0 +fi + +add_bin_path +with_github_cli + +REPO="elastic/integrations" +BRANCH="${BACKPORT_BRANCH_NAME}" +PKG="${PACKAGE_NAME}" +VERSION="${PACKAGE_VERSION}" + +BODY_FILE="$(mktemp)" +trap 'rm -f "${BODY_FILE}"' EXIT + +if [[ "${NOTIFY_STATUS}" == "success" ]]; then + cat > "${BODY_FILE}" < "${BODY_FILE}" < Date: Wed, 17 Jun 2026 12:32:41 +0200 Subject: [PATCH 04/23] [Backport pipeline] Use meta_data in trigger steps and notify script Switch trigger_backport_create.sh and trigger_backport_dryrun.sh to pass values via meta_data instead of env in their trigger steps, so that backport_branch.sh receives them through buildkite-agent meta-data get, consistent with the input-variables step. Update notify_backport_pr.sh to read BACKPORT_BRANCH_NAME, PACKAGE_NAME, PACKAGE_VERSION, and PR_NUMBER from metadata with env var fallbacks. Co-authored-by: Claude Sonnet 4.6 --- .buildkite/scripts/notify_backport_pr.sh | 7 ++++++- .buildkite/scripts/trigger_backport_create.sh | 2 +- .buildkite/scripts/trigger_backport_dryrun.sh | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/notify_backport_pr.sh b/.buildkite/scripts/notify_backport_pr.sh index 381b0613002..e9d4421fb06 100644 --- a/.buildkite/scripts/notify_backport_pr.sh +++ b/.buildkite/scripts/notify_backport_pr.sh @@ -9,7 +9,12 @@ source .buildkite/scripts/common.sh set -euo pipefail -if [[ -z "${PR_NUMBER:-}" ]]; then +BACKPORT_BRANCH_NAME="$(buildkite-agent meta-data get BACKPORT_BRANCH_NAME --default "${BACKPORT_BRANCH_NAME:-""}")" +PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default "${PACKAGE_NAME:-""}")" +PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default "${PACKAGE_VERSION:-""}")" +PR_NUMBER="$(buildkite-agent meta-data get PR_NUMBER --default "${PR_NUMBER:-""}")" + +if [[ -z "${PR_NUMBER}" ]]; then echo "PR_NUMBER not set, skipping PR notification" exit 0 fi diff --git a/.buildkite/scripts/trigger_backport_create.sh b/.buildkite/scripts/trigger_backport_create.sh index fe11b6b8ad8..0055af3d1bd 100644 --- a/.buildkite/scripts/trigger_backport_create.sh +++ b/.buildkite/scripts/trigger_backport_create.sh @@ -106,7 +106,7 @@ while IFS= read -r branch; do - label: ":git: Backport create: ${branch}" trigger: "integrations-backport" build: - env: + meta_data: DRY_RUN: "false" PACKAGE_NAME: "${pkg}" PACKAGE_VERSION: "${base_version}" diff --git a/.buildkite/scripts/trigger_backport_dryrun.sh b/.buildkite/scripts/trigger_backport_dryrun.sh index 5405fd99013..200a56d7346 100755 --- a/.buildkite/scripts/trigger_backport_dryrun.sh +++ b/.buildkite/scripts/trigger_backport_dryrun.sh @@ -106,7 +106,7 @@ while IFS= read -r branch; do - label: ":git: Backport dry-run: ${branch}" trigger: "integrations-backport" build: - env: + meta_data: DRY_RUN: "true" PACKAGE_NAME: "${pkg}" PACKAGE_VERSION: "${base_version}" From b74ad252ec5ac66ba7bee03da032d6c7d7091f7f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jun 2026 15:34:58 +0200 Subject: [PATCH 05/23] [Backport pipeline] Extract shared lib and add tests for trigger scripts Extract the pipeline-generation loop from trigger_backport_dryrun.sh and trigger_backport_create.sh into a shared trigger_backport_lib.sh. Both scripts now source the lib, wrap their context-specific setup in main(), and guard execution with a BASH_SOURCE check to allow sourcing for tests. Add test_trigger_backport.sh with 18 assertions covering dry-run and create modes, PR_NUMBER inclusion, existing/inactive entry skipping, mage error propagation, multiple entries, and invalid inventory handling. Add assert_file_contains/assert_file_not_contains helpers to test_helpers.sh and register the new suite in run_buildkite_scripts_tests.sh. Co-authored-by: Claude Sonnet 4.6 --- .../scripts/run_buildkite_scripts_tests.sh | 4 + .buildkite/scripts/test_helpers.sh | 26 +++ .buildkite/scripts/test_trigger_backport.sh | 202 ++++++++++++++++++ .buildkite/scripts/trigger_backport_create.sh | 162 +++++--------- .buildkite/scripts/trigger_backport_dryrun.sh | 156 +++++--------- .buildkite/scripts/trigger_backport_lib.sh | 87 ++++++++ 6 files changed, 423 insertions(+), 214 deletions(-) create mode 100755 .buildkite/scripts/test_trigger_backport.sh create mode 100644 .buildkite/scripts/trigger_backport_lib.sh diff --git a/.buildkite/scripts/run_buildkite_scripts_tests.sh b/.buildkite/scripts/run_buildkite_scripts_tests.sh index d9ecee694b7..948b41ecb09 100755 --- a/.buildkite/scripts/run_buildkite_scripts_tests.sh +++ b/.buildkite/scripts/run_buildkite_scripts_tests.sh @@ -44,3 +44,7 @@ deactivate echo "" echo "=== Running check_changelog_entries.sh tests ===" run_tests_if_exists "${REPO_ROOT}/.buildkite/scripts/test_check_changelog_entries.sh" + +echo "" +echo "=== Running trigger_backport_lib.sh tests ===" +run_tests_if_exists "${REPO_ROOT}/.buildkite/scripts/test_trigger_backport.sh" diff --git a/.buildkite/scripts/test_helpers.sh b/.buildkite/scripts/test_helpers.sh index a31b51e452a..b1e0a49764e 100644 --- a/.buildkite/scripts/test_helpers.sh +++ b/.buildkite/scripts/test_helpers.sh @@ -27,3 +27,29 @@ assert_exit_code() { (( fail++ )) || true fi } + +assert_file_contains() { + local description="$1" + local needle="$2" + local file="$3" + if grep -qF "${needle}" "${file}" 2>/dev/null; then + echo "PASS: ${description}" + (( pass++ )) || true + else + echo "FAIL: ${description} — '${needle}' not found in ${file}" + (( fail++ )) || true + fi +} + +assert_file_not_contains() { + local description="$1" + local needle="$2" + local file="$3" + if ! grep -qF "${needle}" "${file}" 2>/dev/null; then + echo "PASS: ${description}" + (( pass++ )) || true + else + echo "FAIL: ${description} — '${needle}' unexpectedly found in ${file}" + (( fail++ )) || true + fi +} diff --git a/.buildkite/scripts/test_trigger_backport.sh b/.buildkite/scripts/test_trigger_backport.sh new file mode 100755 index 00000000000..5ee3206481a --- /dev/null +++ b/.buildkite/scripts/test_trigger_backport.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# Unit tests for generate_trigger_pipeline() from trigger_backport_lib.sh. +# Run directly or via .buildkite/scripts/run_buildkite_scripts_tests.sh. +# +# Requires: yq (installed via with_yq in CI; must be available locally too). + +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" + +source "${REPO_ROOT}/.buildkite/scripts/trigger_backport_lib.sh" +source "${REPO_ROOT}/.buildkite/scripts/test_helpers.sh" + +pass=0 +fail=0 + +# --------------------------------------------------------------------------- +# Mock: mage — set MOCK_MAGE_EXIT to control CheckBackportBranchActive result. +# 0 = active (default) +# 1 = inactive +# 2 = error +# --------------------------------------------------------------------------- +mage() { + return "${MOCK_MAGE_EXIT:-0}" +} +MOCK_MAGE_EXIT=0 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +WORK_DIR="" +cleanup() { + [[ -n "${WORK_DIR}" ]] && rm -rf "${WORK_DIR}" +} +trap cleanup EXIT +WORK_DIR="$(mktemp -d)" + +# Write a .backports.yml with one or more entries. +# Each entry requires exactly 5 args: branch pkg base_version base_commit archived +# Usage: write_inventory ... +write_inventory() { + local file="$1"; shift + printf 'backports:\n' > "${file}" + while [[ $# -ge 5 ]]; do + local branch="$1" pkg="$2" base_version="$3" base_commit="$4" archived="$5" + cat >> "${file}" < +run_generate() { + local old="$1" new="$2" dry_run="$3" pr_number="$4" out="$5" + : > "${out}" + generate_trigger_pipeline "${old}" "${new}" "${dry_run}" "${pr_number}" "${out}" || true +} + +OLD="${WORK_DIR}/old.yml" +NEW="${WORK_DIR}/new.yml" +OUT="${WORK_DIR}/pipeline.yml" + +# --------------------------------------------------------------------------- +# Test: new entry in dry-run mode +# --------------------------------------------------------------------------- +echo "--- new entry — dry-run mode" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" + +assert_file_contains "dry-run: step header present" "steps:" "${OUT}" +assert_file_contains "dry-run: label is Backport dry-run" 'Backport dry-run: backport-aws-2.0' "${OUT}" +assert_file_contains "dry-run: DRY_RUN is true" 'DRY_RUN: "true"' "${OUT}" +assert_file_contains "dry-run: PACKAGE_NAME correct" 'PACKAGE_NAME: "aws"' "${OUT}" +assert_file_contains "dry-run: PACKAGE_VERSION correct" 'PACKAGE_VERSION: "2.0.0"' "${OUT}" +assert_file_contains "dry-run: BASE_COMMIT correct" 'BASE_COMMIT: "def456"' "${OUT}" +assert_file_contains "dry-run: BACKPORT_BRANCH_NAME correct" 'BACKPORT_BRANCH_NAME: "backport-aws-2.0"' "${OUT}" +assert_file_not_contains "dry-run: PR_NUMBER absent when empty" "PR_NUMBER" "${OUT}" + +# --------------------------------------------------------------------------- +# Test: new entry in create mode +# --------------------------------------------------------------------------- +echo "--- new entry — create mode" + +run_generate "${OLD}" "${NEW}" "false" "" "${OUT}" + +assert_file_contains "create: label is Backport create" 'Backport create: backport-aws-2.0' "${OUT}" +assert_file_contains "create: DRY_RUN is false" 'DRY_RUN: "false"' "${OUT}" + +# --------------------------------------------------------------------------- +# Test: PR_NUMBER included when provided, omitted when empty +# --------------------------------------------------------------------------- +echo "--- PR_NUMBER" + +run_generate "${OLD}" "${NEW}" "false" "9999" "${OUT}" +assert_file_contains "PR_NUMBER included when set" 'PR_NUMBER: "9999"' "${OUT}" + +run_generate "${OLD}" "${NEW}" "false" "" "${OUT}" +assert_file_not_contains "PR_NUMBER omitted when empty" "PR_NUMBER" "${OUT}" + +# --------------------------------------------------------------------------- +# Test: existing entry is skipped +# --------------------------------------------------------------------------- +echo "--- existing entry skipped" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" + +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" +assert_equals "existing entry: no steps generated" "" "$(cat "${OUT}")" + +# --------------------------------------------------------------------------- +# Test: inactive entry (mage returns 1) is skipped +# --------------------------------------------------------------------------- +echo "--- inactive entry skipped" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "true" + +MOCK_MAGE_EXIT=1 +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" +MOCK_MAGE_EXIT=0 + +assert_equals "inactive entry: no steps generated" "" "$(cat "${OUT}")" + +# --------------------------------------------------------------------------- +# Test: mage error (exit code 2) causes function to return non-zero +# --------------------------------------------------------------------------- +echo "--- mage error propagated" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +MOCK_MAGE_EXIT=2 +err_exit=0 +: > "${OUT}" +generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || err_exit=$? +MOCK_MAGE_EXIT=0 + +assert_exit_code "mage error: non-zero exit returned" "1" "${err_exit}" + +# --------------------------------------------------------------------------- +# Test: multiple new entries all appear in pipeline +# --------------------------------------------------------------------------- +echo "--- multiple new entries" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" \ + "backport-gcp-1.5" "gcp" "1.5.0" "ghi789" "false" + +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" + +assert_file_contains "multiple: aws step present" "backport-aws-2.0" "${OUT}" +assert_file_contains "multiple: gcp step present" "backport-gcp-1.5" "${OUT}" + +# --------------------------------------------------------------------------- +# Test: invalid old inventory returns non-zero +# --------------------------------------------------------------------------- +echo "--- invalid inventory" + +echo "not: yaml: backports" > "${OLD}" +write_inventory "${NEW}" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +inv_exit=0 +: > "${OUT}" +generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || inv_exit=$? + +assert_exit_code "invalid old inventory: non-zero exit" "1" "${inv_exit}" + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo "" +echo "Results: ${pass} passed, ${fail} failed" +if [[ "${fail}" -gt 0 ]]; then + exit 1 +fi diff --git a/.buildkite/scripts/trigger_backport_create.sh b/.buildkite/scripts/trigger_backport_create.sh index 0055af3d1bd..4a18f142c2c 100644 --- a/.buildkite/scripts/trigger_backport_create.sh +++ b/.buildkite/scripts/trigger_backport_create.sh @@ -4,134 +4,78 @@ # runs the integrations-backport pipeline with DRY_RUN=false. source .buildkite/scripts/common.sh +source .buildkite/scripts/trigger_backport_lib.sh set -euo pipefail -if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - echo "Pull request build, skipping backport create trigger" - exit 0 -fi - -if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then - echo "Not on main branch (branch: ${BUILDKITE_BRANCH}), skipping backport create trigger" - exit 0 -fi - -add_bin_path -with_yq -with_mage -with_github_cli - -if ! git diff --name-only HEAD^ HEAD | grep -qE '^\.backports\.yml$'; then - echo ".backports.yml not changed, skipping backport create trigger" - exit 0 -fi - -echo "--- .backports.yml changed — finding new entries" - -OLD_INVENTORY="" -PIPELINE_FILE="" - -cleanup() { - local exit_code=$? - [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" - [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" - exit "${exit_code}" -} -trap cleanup EXIT - -PR_NUMBER=$(gh api "repos/elastic/integrations/commits/${BUILDKITE_COMMIT}/pulls" --jq '.[0].number' 2>/dev/null || true) -if [[ -n "${PR_NUMBER}" ]]; then - echo "Associated PR: #${PR_NUMBER}" -else - echo "Could not resolve PR number for commit ${BUILDKITE_COMMIT}, PR comments will be skipped" -fi +main() { + if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then + echo "Pull request build, skipping backport create trigger" + exit 0 + fi -OLD_INVENTORY="$(mktemp)" -NEW_INVENTORY=".backports.yml" + if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then + echo "Not on main branch (branch: ${BUILDKITE_BRANCH}), skipping backport create trigger" + exit 0 + fi -if ! git show "HEAD^:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then - echo ".backports.yml is new on main — skipping create for initial entries" - echo "To create branches for these entries, trigger the integrations-backport pipeline manually." - exit 0 -fi + add_bin_path + with_yq + with_mage + with_github_cli -if ! yq -e '.backports' "${OLD_INVENTORY}" > /dev/null; then - echo "ERROR: old inventory is not valid YAML or missing 'backports' key: ${OLD_INVENTORY}" - exit 1 -fi + if ! git diff --name-only HEAD^ HEAD | grep -qE '^\.backports\.yml$'; then + echo ".backports.yml not changed, skipping backport create trigger" + exit 0 + fi -if ! yq -e '.backports' "${NEW_INVENTORY}" > /dev/null; then - echo "ERROR: new inventory is not valid YAML or missing 'backports' key: ${NEW_INVENTORY}" - exit 1 -fi + echo "--- .backports.yml changed — finding new entries" -PIPELINE_FILE="$(mktemp --suffix=.yml)" -entries_found=0 + OLD_INVENTORY="" + PIPELINE_FILE="" -while IFS= read -r branch; do - entry=".backports[] | select(.branch == \"${branch}\")" + cleanup() { + local exit_code=$? + [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" + [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" + exit "${exit_code}" + } + trap cleanup EXIT - active_exit=0 - mage CheckBackportBranchActive "${branch}" || active_exit=$? - if [[ "${active_exit}" -eq 2 ]]; then - echo "ERROR: failed to check active status for branch '${branch}'" - exit 1 - fi - if [[ "${active_exit}" -ne 0 ]]; then - echo " Skipping inactive entry: ${branch}" - continue + PR_NUMBER=$(gh api "repos/elastic/integrations/commits/${BUILDKITE_COMMIT}/pulls" --jq '.[0].number' 2>/dev/null || true) + if [[ -n "${PR_NUMBER}" ]]; then + echo "Associated PR: #${PR_NUMBER}" + else + echo "Could not resolve PR number for commit ${BUILDKITE_COMMIT}, PR comments will be skipped" fi - # Only trigger for entries that are new (absent from the pre-merge state). - # If the entry already existed, the branch is already created; nothing to provision. - old_branch="$(yq "${entry} | .branch" "${OLD_INVENTORY}")" + OLD_INVENTORY="$(mktemp)" + NEW_INVENTORY=".backports.yml" - if [[ -n "${old_branch}" ]]; then - echo " Skipping existing entry: ${branch} (already present before merge)" - continue + if ! git show "HEAD^:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then + echo ".backports.yml is new on main — skipping create for initial entries" + echo "To create branches for these entries, trigger the integrations-backport pipeline manually." + exit 0 fi - pkg="$(yq "${entry} | .package" "${NEW_INVENTORY}")" - base_version="$(yq "${entry} | .base_version" "${NEW_INVENTORY}")" - base_commit="$(yq "${entry} | .base_commit" "${NEW_INVENTORY}")" + PIPELINE_FILE="$(mktemp --suffix=.yml)" - echo " Queuing create: ${branch} (package=${pkg} version=${base_version} base_commit=${base_commit})" + generate_trigger_pipeline "${OLD_INVENTORY}" "${NEW_INVENTORY}" "false" "${PR_NUMBER}" "${PIPELINE_FILE}" - if [[ "${entries_found}" -eq 0 ]]; then - printf 'steps:\n' > "${PIPELINE_FILE}" - fi + rm -f "${OLD_INVENTORY}" - cat >> "${PIPELINE_FILE}" <> "${PIPELINE_FILE}" < "${OLD_INVENTORY}" 2>/dev/null; then - echo ".backports.yml is new on ${BASE_BRANCH} — skipping dry-runs for initial entries" - echo "To validate new entries, add them in a follow-up PR after this one merges." - exit 0 -fi - -if ! yq -e '.backports' "${OLD_INVENTORY}" > /dev/null; then - echo "ERROR: old inventory is not valid YAML or missing 'backports' key: ${OLD_INVENTORY}" - exit 1 -fi +main() { + if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then + echo "Not a pull request, skipping backport dry-run trigger" + exit 0 + fi -if ! yq -e '.backports' "${NEW_INVENTORY}" > /dev/null; then - echo "ERROR: new inventory is not valid YAML or missing 'backports' key: ${NEW_INVENTORY}" - exit 1 -fi + if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "main" ]]; then + echo "Pull request does not target main (base branch: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}), skipping backport dry-run trigger" + exit 0 + fi -PIPELINE_FILE="$(mktemp --suffix=.yml)" -entries_found=0 + add_bin_path + with_yq + with_mage -while IFS= read -r branch; do - entry=".backports[] | select(.branch == \"${branch}\")" + from="$(get_from_changeset)" + to="$(get_to_changeset)" + commit_merge="$(git merge-base "${from}" "${to}")" - active_exit=0 - mage CheckBackportBranchActive "${branch}" || active_exit=$? - if [[ "${active_exit}" -eq 2 ]]; then - echo "ERROR: failed to check active status for branch '${branch}'" - exit 1 - fi - if [[ "${active_exit}" -ne 0 ]]; then - echo " Skipping inactive entry: ${branch}" - continue + if ! git diff --name-only "${commit_merge}" "${to}" | grep -qE '^\.backports\.yml$'; then + echo ".backports.yml not changed, skipping backport dry-run trigger" + exit 0 fi - # Only trigger for entries that are new (absent from the old inventory). - # If the entry already existed, the git branch is already created; there is - # nothing to provision. This also covers re-activating a previously archived - # entry (archived:true → false): the branch exists, so no dry-run is needed. - old_branch="$(yq "${entry} | .branch" "${OLD_INVENTORY}")" + echo "--- .backports.yml changed — finding new entries" - if [[ -n "${old_branch}" ]]; then - echo " Skipping existing entry: ${branch} (already present in base branch)" - continue - fi + BASE_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" + OLD_INVENTORY="" + PIPELINE_FILE="" - pkg="$(yq "${entry} | .package" "${NEW_INVENTORY}")" - base_version="$(yq "${entry} | .base_version" "${NEW_INVENTORY}")" - base_commit="$(yq "${entry} | .base_commit" "${NEW_INVENTORY}")" + cleanup() { + local exit_code=$? + [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" + [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" + exit "${exit_code}" + } + trap cleanup EXIT - echo " Queuing dry-run: ${branch} (package=${pkg} version=${base_version} base_commit=${base_commit})" + OLD_INVENTORY="$(mktemp)" + NEW_INVENTORY=".backports.yml" - if [[ "${entries_found}" -eq 0 ]]; then - printf 'steps:\n' > "${PIPELINE_FILE}" + if ! git show "origin/${BASE_BRANCH}:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then + echo ".backports.yml is new on ${BASE_BRANCH} — skipping dry-runs for initial entries" + echo "To validate new entries, add them in a follow-up PR after this one merges." + exit 0 fi - cat >> "${PIPELINE_FILE}" < /dev/null 2>&1; then + echo "ERROR: old inventory is not valid YAML or missing 'backports' key: ${old_inventory}" >&2 + return 1 + fi + + if ! yq -e '.backports' "${new_inventory}" > /dev/null 2>&1; then + echo "ERROR: new inventory is not valid YAML or missing 'backports' key: ${new_inventory}" >&2 + return 1 + fi + + local label_prefix + if [[ "${dry_run}" == "true" ]]; then + label_prefix="Backport dry-run" + else + label_prefix="Backport create" + fi + + while IFS= read -r branch; do + local entry=".backports[] | select(.branch == \"${branch}\")" + + local active_exit=0 + mage CheckBackportBranchActive "${branch}" || active_exit=$? + if [[ "${active_exit}" -eq 2 ]]; then + echo "ERROR: failed to check active status for branch '${branch}'" >&2 + return 1 + fi + if [[ "${active_exit}" -ne 0 ]]; then + echo " Skipping inactive entry: ${branch}" + continue + fi + + # Only trigger for entries that are new (absent from the old inventory). + # If the entry already existed, the branch is already created; nothing to provision. + # This also covers re-activating a previously archived entry (archived:true → false): + # the branch exists, so no trigger is needed. + local old_branch + old_branch="$(yq "${entry} | .branch" "${old_inventory}")" + if [[ -n "${old_branch}" ]]; then + echo " Skipping existing entry: ${branch} (already present)" + continue + fi + + local pkg base_version base_commit + pkg="$(yq "${entry} | .package" "${new_inventory}")" + base_version="$(yq "${entry} | .base_version" "${new_inventory}")" + base_commit="$(yq "${entry} | .base_commit" "${new_inventory}")" + + echo " Queuing ${label_prefix}: ${branch} (package=${pkg} version=${base_version} base_commit=${base_commit})" + + if [[ ! -s "${pipeline_file}" ]]; then + printf 'steps:\n' > "${pipeline_file}" + fi + + cat >> "${pipeline_file}" <> "${pipeline_file}" < Date: Wed, 17 Jun 2026 15:46:24 +0200 Subject: [PATCH 06/23] [Backport pipeline] Register new backport scripts in CI trigger patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add notify_backport_pr.sh, trigger_backport_create.sh, trigger_backport_lib.sh, and test_trigger_backport.sh to non_package_patterns in common.sh so CI recognises changes to these files as non-package modifications. Add notify_backport_pr.sh to skip_ci_on_only_changed in pull-requests.json — PRs touching only this script (no unit tests) do not need a full CI run. Co-Authored-By: Claude Sonnet 4.6 --- .buildkite/pull-requests.json | 1 + .buildkite/scripts/common.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index a46f45b5172..1adc1790780 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -23,6 +23,7 @@ "^.buildkite/pull-requests.json$", "^.buildkite/scripts/backport_branch.sh$", "^.buildkite/scripts/build_packages.sh$", + "^.buildkite/scripts/notify_backport_pr.sh$", "^.github/dependabot.yml$", "^.github/workflows/", "^.github/stale.yml$", diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7a7640ad7d8..0aab883b6f0 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -789,7 +789,10 @@ is_pr_affected() { '\.buildkite/pull-requests\.json' '\.buildkite/scripts/backport_branch\.sh' '\.buildkite/scripts/check_backports_inventory\.sh' + '\.buildkite/scripts/notify_backport_pr\.sh' + '\.buildkite/scripts/trigger_backport_create\.sh' '\.buildkite/scripts/trigger_backport_dryrun\.sh' + '\.buildkite/scripts/trigger_backport_lib\.sh' '\.buildkite/scripts/build_packages\.sh' '\.buildkite/scripts/check_changelog_entries\.sh' '\.buildkite/scripts/packages/.+\.sh' @@ -798,6 +801,7 @@ is_pr_affected() { '\.buildkite/scripts/run_dev_scripts_tests\.sh' '\.buildkite/scripts/test_check_changelog_entries\.sh' '\.buildkite/scripts/test_helpers\.sh' + '\.buildkite/scripts/test_trigger_backport\.sh' '\.github/dependabot\.yml' '\.github/stale\.yml' '\.github/workflows/' From c4a2fd03e2e7113c1445ad8366f6a63f5d1a3750 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jun 2026 15:54:33 +0200 Subject: [PATCH 07/23] [Backport pipeline] Extract helper functions into trigger_backport_lib.sh Move three inline snippets from trigger_backport_dryrun.sh and trigger_backport_create.sh into dedicated functions in the shared lib: - backports_yml_changed(from, to): git diff check for .backports.yml - load_old_backports_inventory(ref, file): git show at a given ref - resolve_pr_number(commit): GitHub API lookup, logs to stderr, echoes the number to stdout for capture Both trigger scripts are leaner; the logic is now testable in isolation. Co-authored-by: Claude Sonnet 4.6 --- .buildkite/scripts/trigger_backport_create.sh | 11 ++----- .buildkite/scripts/trigger_backport_dryrun.sh | 4 +-- .buildkite/scripts/trigger_backport_lib.sh | 31 ++++++++++++++++++- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.buildkite/scripts/trigger_backport_create.sh b/.buildkite/scripts/trigger_backport_create.sh index 4a18f142c2c..ede44a13c7c 100644 --- a/.buildkite/scripts/trigger_backport_create.sh +++ b/.buildkite/scripts/trigger_backport_create.sh @@ -24,7 +24,7 @@ main() { with_mage with_github_cli - if ! git diff --name-only HEAD^ HEAD | grep -qE '^\.backports\.yml$'; then + if ! backports_yml_changed "HEAD^" "HEAD"; then echo ".backports.yml not changed, skipping backport create trigger" exit 0 fi @@ -42,17 +42,12 @@ main() { } trap cleanup EXIT - PR_NUMBER=$(gh api "repos/elastic/integrations/commits/${BUILDKITE_COMMIT}/pulls" --jq '.[0].number' 2>/dev/null || true) - if [[ -n "${PR_NUMBER}" ]]; then - echo "Associated PR: #${PR_NUMBER}" - else - echo "Could not resolve PR number for commit ${BUILDKITE_COMMIT}, PR comments will be skipped" - fi + PR_NUMBER=$(resolve_pr_number "${BUILDKITE_COMMIT}") OLD_INVENTORY="$(mktemp)" NEW_INVENTORY=".backports.yml" - if ! git show "HEAD^:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then + if ! load_old_backports_inventory "HEAD^" "${OLD_INVENTORY}"; then echo ".backports.yml is new on main — skipping create for initial entries" echo "To create branches for these entries, trigger the integrations-backport pipeline manually." exit 0 diff --git a/.buildkite/scripts/trigger_backport_dryrun.sh b/.buildkite/scripts/trigger_backport_dryrun.sh index 6524726f1f7..5553b901c60 100755 --- a/.buildkite/scripts/trigger_backport_dryrun.sh +++ b/.buildkite/scripts/trigger_backport_dryrun.sh @@ -27,7 +27,7 @@ main() { to="$(get_to_changeset)" commit_merge="$(git merge-base "${from}" "${to}")" - if ! git diff --name-only "${commit_merge}" "${to}" | grep -qE '^\.backports\.yml$'; then + if ! backports_yml_changed "${commit_merge}" "${to}"; then echo ".backports.yml not changed, skipping backport dry-run trigger" exit 0 fi @@ -49,7 +49,7 @@ main() { OLD_INVENTORY="$(mktemp)" NEW_INVENTORY=".backports.yml" - if ! git show "origin/${BASE_BRANCH}:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then + if ! load_old_backports_inventory "origin/${BASE_BRANCH}" "${OLD_INVENTORY}"; then echo ".backports.yml is new on ${BASE_BRANCH} — skipping dry-runs for initial entries" echo "To validate new entries, add them in a follow-up PR after this one merges." exit 0 diff --git a/.buildkite/scripts/trigger_backport_lib.sh b/.buildkite/scripts/trigger_backport_lib.sh index c36798728e8..af9f3bc684d 100644 --- a/.buildkite/scripts/trigger_backport_lib.sh +++ b/.buildkite/scripts/trigger_backport_lib.sh @@ -2,10 +2,39 @@ # Shared library for trigger_backport_dryrun.sh and trigger_backport_create.sh. # Source this file — do not execute directly. # -# Provides generate_trigger_pipeline(), which iterates over a new .backports.yml, +# Provides resolve_pr_number(), backports_yml_changed(), load_old_backports_inventory(), +# and generate_trigger_pipeline(). +# resolve_pr_number() looks up the PR that introduced a given commit via the GitHub API. +# backports_yml_changed() checks whether .backports.yml was modified between two git refs. +# load_old_backports_inventory() extracts .backports.yml at a given git ref into a file. +# generate_trigger_pipeline() iterates over a new .backports.yml, # finds entries absent from the old inventory, and writes Buildkite trigger steps # to a pipeline file. The caller decides whether to upload the result. +resolve_pr_number() { + local commit="$1" + local pr_number + pr_number=$(gh api "repos/elastic/integrations/commits/${commit}/pulls" --jq '.[0].number' 2>/dev/null || true) + if [[ -n "${pr_number}" ]]; then + echo "Associated PR: #${pr_number}" >&2 + else + echo "Could not resolve PR number for commit ${commit}, PR comments will be skipped" >&2 + fi + echo "${pr_number}" +} + +load_old_backports_inventory() { + local git_ref="$1" + local output_file="$2" + git show "${git_ref}:.backports.yml" > "${output_file}" 2>/dev/null +} + +backports_yml_changed() { + local from="$1" + local to="$2" + git diff --name-only "${from}" "${to}" | grep -E '^\.backports\.yml$' > /dev/null +} + generate_trigger_pipeline() { local old_inventory="$1" # path to the pre-change inventory local new_inventory="$2" # path to the post-change inventory From f724bd1810bbf3d8e1ca5108b0b2f3f8e0d058e6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jun 2026 16:04:51 +0200 Subject: [PATCH 08/23] [Backport pipeline] Merge trigger scripts into a single trigger_backport.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace trigger_backport_dryrun.sh and trigger_backport_create.sh with a single trigger_backport.sh. The script detects its mode from BUILDKITE_PULL_REQUEST: PR builds run in dry-run mode against the base branch; post-merge builds on main create the branches. Both pipeline.yml steps continue to express their own guards and dependencies — the shared script handles the rest via the extracted lib functions. Co-authored-by: Claude Sonnet 4.6 --- .buildkite/pipeline.yml | 4 +- .buildkite/scripts/common.sh | 3 +- .buildkite/scripts/trigger_backport.sh | 97 +++++++++++++++++++ .buildkite/scripts/trigger_backport_create.sh | 76 --------------- .buildkite/scripts/trigger_backport_dryrun.sh | 78 --------------- 5 files changed, 100 insertions(+), 158 deletions(-) create mode 100755 .buildkite/scripts/trigger_backport.sh delete mode 100644 .buildkite/scripts/trigger_backport_create.sh delete mode 100755 .buildkite/scripts/trigger_backport_dryrun.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index cd4e30e2472..7cb180bed27 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -86,7 +86,7 @@ steps: - label: ":git: Trigger backport dry-runs" key: "trigger-backport-dryrun" - command: ".buildkite/scripts/trigger_backport_dryrun.sh" + command: ".buildkite/scripts/trigger_backport.sh" agents: image: "${LINUX_AGENT_IMAGE}" depends_on: @@ -101,7 +101,7 @@ steps: - label: ":git: Create backport branches for new entries" key: "trigger-backport-create" - command: ".buildkite/scripts/trigger_backport_create.sh" + command: ".buildkite/scripts/trigger_backport.sh" agents: image: "${LINUX_AGENT_IMAGE}" plugins: diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 0aab883b6f0..e5324c7d268 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -790,8 +790,7 @@ is_pr_affected() { '\.buildkite/scripts/backport_branch\.sh' '\.buildkite/scripts/check_backports_inventory\.sh' '\.buildkite/scripts/notify_backport_pr\.sh' - '\.buildkite/scripts/trigger_backport_create\.sh' - '\.buildkite/scripts/trigger_backport_dryrun\.sh' + '\.buildkite/scripts/trigger_backport\.sh' '\.buildkite/scripts/trigger_backport_lib\.sh' '\.buildkite/scripts/build_packages\.sh' '\.buildkite/scripts/check_changelog_entries\.sh' diff --git a/.buildkite/scripts/trigger_backport.sh b/.buildkite/scripts/trigger_backport.sh new file mode 100755 index 00000000000..3f8ae3c5662 --- /dev/null +++ b/.buildkite/scripts/trigger_backport.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Triggers the integrations-backport pipeline when .backports.yml changes. +# In PR builds: validates new entries via a dry run against the base branch. +# On push to main: creates backport branches for new entries. + +source .buildkite/scripts/common.sh +source .buildkite/scripts/trigger_backport_lib.sh + +set -euo pipefail + +main() { + add_bin_path + with_yq + with_mage + + local dry_run old_inventory_ref diff_from diff_to pr_number="" label + local new_entry_msg new_entry_hint + + if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then + if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "main" ]]; then + echo "Pull request does not target main (base branch: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}), skipping" + exit 0 + fi + dry_run="true" + label="dry-run" + local from to + from="$(get_from_changeset)" + to="$(get_to_changeset)" + diff_from="$(git merge-base "${from}" "${to}")" + diff_to="${to}" + old_inventory_ref="origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" + new_entry_msg="skipping dry-runs for initial entries" + new_entry_hint="To validate new entries, add them in a follow-up PR after this one merges." + else + if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then + echo "Not on main branch (branch: ${BUILDKITE_BRANCH}), skipping" + exit 0 + fi + dry_run="false" + label="create" + diff_from="HEAD^" + diff_to="HEAD" + old_inventory_ref="HEAD^" + new_entry_msg="skipping create for initial entries" + new_entry_hint="To create branches for these entries, trigger the integrations-backport pipeline manually." + with_github_cli + pr_number="$(resolve_pr_number "${BUILDKITE_COMMIT}")" + fi + + if ! backports_yml_changed "${diff_from}" "${diff_to}"; then + echo ".backports.yml not changed, skipping backport ${label} trigger" + exit 0 + fi + + echo "--- .backports.yml changed — finding new entries" + + OLD_INVENTORY="" + PIPELINE_FILE="" + + cleanup() { + local exit_code=$? + [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" + [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" + exit "${exit_code}" + } + trap cleanup EXIT + + OLD_INVENTORY="$(mktemp)" + NEW_INVENTORY=".backports.yml" + + if ! load_old_backports_inventory "${old_inventory_ref}" "${OLD_INVENTORY}"; then + echo ".backports.yml is new — ${new_entry_msg}" + echo "${new_entry_hint}" + exit 0 + fi + + PIPELINE_FILE="$(mktemp --suffix=.yml)" + + generate_trigger_pipeline "${OLD_INVENTORY}" "${NEW_INVENTORY}" "${dry_run}" "${pr_number}" "${PIPELINE_FILE}" + + rm -f "${OLD_INVENTORY}" + + if [[ ! -s "${PIPELINE_FILE}" ]]; then + echo "No new non-archived entries found, skipping ${label} trigger" + rm -f "${PIPELINE_FILE}" + exit 0 + fi + + echo "--- Uploading ${label} trigger(s)" + cat "${PIPELINE_FILE}" + buildkite-agent pipeline upload "${PIPELINE_FILE}" + rm -f "${PIPELINE_FILE}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi diff --git a/.buildkite/scripts/trigger_backport_create.sh b/.buildkite/scripts/trigger_backport_create.sh deleted file mode 100644 index ede44a13c7c..00000000000 --- a/.buildkite/scripts/trigger_backport_create.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# Triggered from the main pipeline when .backports.yml changes on a push to main. -# For each entry that is new (absent from HEAD^), uploads a trigger step that -# runs the integrations-backport pipeline with DRY_RUN=false. - -source .buildkite/scripts/common.sh -source .buildkite/scripts/trigger_backport_lib.sh - -set -euo pipefail - -main() { - if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - echo "Pull request build, skipping backport create trigger" - exit 0 - fi - - if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then - echo "Not on main branch (branch: ${BUILDKITE_BRANCH}), skipping backport create trigger" - exit 0 - fi - - add_bin_path - with_yq - with_mage - with_github_cli - - if ! backports_yml_changed "HEAD^" "HEAD"; then - echo ".backports.yml not changed, skipping backport create trigger" - exit 0 - fi - - echo "--- .backports.yml changed — finding new entries" - - OLD_INVENTORY="" - PIPELINE_FILE="" - - cleanup() { - local exit_code=$? - [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" - [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" - exit "${exit_code}" - } - trap cleanup EXIT - - PR_NUMBER=$(resolve_pr_number "${BUILDKITE_COMMIT}") - - OLD_INVENTORY="$(mktemp)" - NEW_INVENTORY=".backports.yml" - - if ! load_old_backports_inventory "HEAD^" "${OLD_INVENTORY}"; then - echo ".backports.yml is new on main — skipping create for initial entries" - echo "To create branches for these entries, trigger the integrations-backport pipeline manually." - exit 0 - fi - - PIPELINE_FILE="$(mktemp --suffix=.yml)" - - generate_trigger_pipeline "${OLD_INVENTORY}" "${NEW_INVENTORY}" "false" "${PR_NUMBER}" "${PIPELINE_FILE}" - - rm -f "${OLD_INVENTORY}" - - if [[ ! -s "${PIPELINE_FILE}" ]]; then - echo "No new non-archived entries found, skipping create trigger" - rm -f "${PIPELINE_FILE}" - exit 0 - fi - - echo "--- Uploading create trigger(s)" - cat "${PIPELINE_FILE}" - buildkite-agent pipeline upload "${PIPELINE_FILE}" - rm -f "${PIPELINE_FILE}" -} - -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main -fi diff --git a/.buildkite/scripts/trigger_backport_dryrun.sh b/.buildkite/scripts/trigger_backport_dryrun.sh deleted file mode 100755 index 5553b901c60..00000000000 --- a/.buildkite/scripts/trigger_backport_dryrun.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Triggered from the main pipeline when .backports.yml changes in a PR. -# For each entry that is new (absent from the base branch), uploads a trigger -# step that runs the integrations-backport pipeline in DRY_RUN mode. - -source .buildkite/scripts/common.sh -source .buildkite/scripts/trigger_backport_lib.sh - -set -euo pipefail - -main() { - if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then - echo "Not a pull request, skipping backport dry-run trigger" - exit 0 - fi - - if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "main" ]]; then - echo "Pull request does not target main (base branch: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}), skipping backport dry-run trigger" - exit 0 - fi - - add_bin_path - with_yq - with_mage - - from="$(get_from_changeset)" - to="$(get_to_changeset)" - commit_merge="$(git merge-base "${from}" "${to}")" - - if ! backports_yml_changed "${commit_merge}" "${to}"; then - echo ".backports.yml not changed, skipping backport dry-run trigger" - exit 0 - fi - - echo "--- .backports.yml changed — finding new entries" - - BASE_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" - OLD_INVENTORY="" - PIPELINE_FILE="" - - cleanup() { - local exit_code=$? - [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" - [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" - exit "${exit_code}" - } - trap cleanup EXIT - - OLD_INVENTORY="$(mktemp)" - NEW_INVENTORY=".backports.yml" - - if ! load_old_backports_inventory "origin/${BASE_BRANCH}" "${OLD_INVENTORY}"; then - echo ".backports.yml is new on ${BASE_BRANCH} — skipping dry-runs for initial entries" - echo "To validate new entries, add them in a follow-up PR after this one merges." - exit 0 - fi - - PIPELINE_FILE="$(mktemp --suffix=.yml)" - - generate_trigger_pipeline "${OLD_INVENTORY}" "${NEW_INVENTORY}" "true" "" "${PIPELINE_FILE}" - - rm -f "${OLD_INVENTORY}" - - if [[ ! -s "${PIPELINE_FILE}" ]]; then - echo "No new non-archived entries found, skipping dry-run trigger" - rm -f "${PIPELINE_FILE}" - exit 0 - fi - - echo "--- Uploading dry-run trigger(s)" - cat "${PIPELINE_FILE}" - buildkite-agent pipeline upload "${PIPELINE_FILE}" - rm -f "${PIPELINE_FILE}" -} - -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main -fi From 7c9ebf106c895e3427725ab4c107a832bb9be091 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jun 2026 16:36:17 +0200 Subject: [PATCH 09/23] [Backport pipeline] Fix notify-pr step and trigger lib bugs from code review Co-authored-by: Claude Sonnet 4.6 --- .buildkite/pipeline.backport.yml | 1 - .buildkite/scripts/trigger_backport_lib.sh | 24 +++++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 27038bff122..88860d52bec 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -88,4 +88,3 @@ steps: depends_on: - step: "create-backport-branch" allow_failure: true - if: "build.env('PR_NUMBER') != ''" diff --git a/.buildkite/scripts/trigger_backport_lib.sh b/.buildkite/scripts/trigger_backport_lib.sh index af9f3bc684d..d41fb70def8 100644 --- a/.buildkite/scripts/trigger_backport_lib.sh +++ b/.buildkite/scripts/trigger_backport_lib.sh @@ -14,7 +14,7 @@ resolve_pr_number() { local commit="$1" local pr_number - pr_number=$(gh api "repos/elastic/integrations/commits/${commit}/pulls" --jq '.[0].number' 2>/dev/null || true) + pr_number=$(gh api "repos/elastic/integrations/commits/${commit}/pulls" --jq '.[0].number // empty' 2>/dev/null || true) if [[ -n "${pr_number}" ]]; then echo "Associated PR: #${pr_number}" >&2 else @@ -62,17 +62,6 @@ generate_trigger_pipeline() { while IFS= read -r branch; do local entry=".backports[] | select(.branch == \"${branch}\")" - local active_exit=0 - mage CheckBackportBranchActive "${branch}" || active_exit=$? - if [[ "${active_exit}" -eq 2 ]]; then - echo "ERROR: failed to check active status for branch '${branch}'" >&2 - return 1 - fi - if [[ "${active_exit}" -ne 0 ]]; then - echo " Skipping inactive entry: ${branch}" - continue - fi - # Only trigger for entries that are new (absent from the old inventory). # If the entry already existed, the branch is already created; nothing to provision. # This also covers re-activating a previously archived entry (archived:true → false): @@ -84,6 +73,17 @@ generate_trigger_pipeline() { continue fi + local active_exit=0 + mage CheckBackportBranchActive "${branch}" || active_exit=$? + if [[ "${active_exit}" -eq 2 ]]; then + echo "ERROR: failed to check active status for branch '${branch}'" >&2 + return 1 + fi + if [[ "${active_exit}" -ne 0 ]]; then + echo " Skipping inactive entry: ${branch}" + continue + fi + local pkg base_version base_commit pkg="$(yq "${entry} | .package" "${new_inventory}")" base_version="$(yq "${entry} | .base_version" "${new_inventory}")" From 09c47ba551569ebde1be4d45c700397e8a779213 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jun 2026 17:21:18 +0200 Subject: [PATCH 10/23] [Backport pipeline] Add branch name validation and idempotent PR comment helper Co-Authored-By: Claude Sonnet 4.6 --- .buildkite/scripts/common.sh | 30 +++++++++++++++++++++ .buildkite/scripts/notify_backport_pr.sh | 12 +++------ .buildkite/scripts/test_trigger_backport.sh | 18 +++++++++++++ .buildkite/scripts/trigger_backport_lib.sh | 14 +++++++++- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e5324c7d268..94bfb8adb7c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -1245,6 +1245,36 @@ delete_and_create_gh_pr_comment() { --body "${contents}" } +# Posts a new comment on every pipeline run/retry while staying idempotent +# within a single attempt. Pass a unique id per attempt (e.g. build-number + +# retry-count) — if a comment with that id already exists the call is a no-op, +# so transient gh failures can be retried safely without double-posting. +create_new_gh_pr_comment() { + local owner="$1" + local repo="$2" + local pr_number="$3" + local id="$4" + local comment_file="$5" + local metadata="" + + local comment_id + comment_id=$(get_comment_with_pattern "${owner}" "${repo}" "${pr_number}" "${metadata}") + if [[ -n "${comment_id}" ]]; then + echo "Comment already posted for id=${id}, skipping" + return + fi + + local contents + contents="$(cat "${comment_file}")" + printf -v contents '%s\n%s' "${contents}" "${metadata}" + + echo "Creating new comment" + gh pr comment \ + "${pr_number}" \ + --repo "${owner}/${repo}" \ + --body "${contents}" +} + # FIXME: In a Pull Request that there are more than 100 comments, # if the comment is older than those 100 comments, it won't be found due to pagination get_comment_with_pattern() { diff --git a/.buildkite/scripts/notify_backport_pr.sh b/.buildkite/scripts/notify_backport_pr.sh index e9d4421fb06..5a40e8bb456 100644 --- a/.buildkite/scripts/notify_backport_pr.sh +++ b/.buildkite/scripts/notify_backport_pr.sh @@ -22,26 +22,22 @@ fi add_bin_path with_github_cli -REPO="elastic/integrations" -BRANCH="${BACKPORT_BRANCH_NAME}" -PKG="${PACKAGE_NAME}" -VERSION="${PACKAGE_VERSION}" - BODY_FILE="$(mktemp)" trap 'rm -f "${BODY_FILE}"' EXIT if [[ "${NOTIFY_STATUS}" == "success" ]]; then cat > "${BODY_FILE}" < "${BODY_FILE}" </dev/null; echo $?)" +assert_exit_code "valid: x minor" "0" "$(validate_backport_branch_name "backport-aws-6.x" 2>/dev/null; echo $?)" +assert_exit_code "valid: underscore in pkg" "0" "$(validate_backport_branch_name "backport-my_pkg-2.0" 2>/dev/null; echo $?)" +assert_exit_code "valid: single-digit minor" "0" "$(validate_backport_branch_name "backport-gcp-1.5" 2>/dev/null; echo $?)" + +# Invalid names +assert_exit_code "invalid: hyphen in pkg" "1" "$(validate_backport_branch_name "backport-my-pkg-2.0" 2>/dev/null; echo $?)" +assert_exit_code "invalid: missing minor" "1" "$(validate_backport_branch_name "backport-aws-1" 2>/dev/null; echo $?)" +assert_exit_code "invalid: mixed minor" "1" "$(validate_backport_branch_name "backport-aws-6.1x" 2>/dev/null; echo $?)" +assert_exit_code "invalid: wrong prefix" "1" "$(validate_backport_branch_name "not-backport-1.0" 2>/dev/null; echo $?)" +assert_exit_code "invalid: double-quote" "1" "$(validate_backport_branch_name 'backport-foo"bar-1.0' 2>/dev/null; echo $?)" + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- diff --git a/.buildkite/scripts/trigger_backport_lib.sh b/.buildkite/scripts/trigger_backport_lib.sh index d41fb70def8..d907b93e7fc 100644 --- a/.buildkite/scripts/trigger_backport_lib.sh +++ b/.buildkite/scripts/trigger_backport_lib.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Shared library for trigger_backport_dryrun.sh and trigger_backport_create.sh. +# Shared library for trigger_backport.sh. # Source this file — do not execute directly. # # Provides resolve_pr_number(), backports_yml_changed(), load_old_backports_inventory(), @@ -29,6 +29,14 @@ load_old_backports_inventory() { git show "${git_ref}:.backports.yml" > "${output_file}" 2>/dev/null } +validate_backport_branch_name() { + local branch="$1" + if [[ ! "${branch}" =~ ^backport-[a-zA-Z0-9_]+-[0-9]+\.([0-9]+|x)$ ]]; then + echo "ERROR: invalid branch name '${branch}' — expected format: backport--." >&2 + return 1 + fi +} + backports_yml_changed() { local from="$1" local to="$2" @@ -60,6 +68,10 @@ generate_trigger_pipeline() { fi while IFS= read -r branch; do + if ! validate_backport_branch_name "${branch}"; then + return 1 + fi + local entry=".backports[] | select(.branch == \"${branch}\")" # Only trigger for entries that are new (absent from the old inventory). From 4316998380b6fc286ea90d1ae2de4e2c35290d24 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 10:39:02 +0200 Subject: [PATCH 11/23] [Backport pipeline] Fix error masking in backports_yml_changed, load_old_backports_inventory and resolve_pr_number Co-Authored-By: Claude Sonnet 4.6 --- .buildkite/scripts/trigger_backport.sh | 12 ++++++++++-- .buildkite/scripts/trigger_backport_lib.sh | 20 ++++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/trigger_backport.sh b/.buildkite/scripts/trigger_backport.sh index 3f8ae3c5662..3a8b1da92c1 100755 --- a/.buildkite/scripts/trigger_backport.sh +++ b/.buildkite/scripts/trigger_backport.sh @@ -44,10 +44,18 @@ main() { new_entry_msg="skipping create for initial entries" new_entry_hint="To create branches for these entries, trigger the integrations-backport pipeline manually." with_github_cli - pr_number="$(resolve_pr_number "${BUILDKITE_COMMIT}")" + pr_number="" + pr_number="$(retry 3 resolve_pr_number "${BUILDKITE_COMMIT}")" || { + echo "Warning: could not resolve PR number after retries, PR comments will be skipped" >&2 + } fi - if ! backports_yml_changed "${diff_from}" "${diff_to}"; then + backports_yml_changed_exit=0 + backports_yml_changed "${diff_from}" "${diff_to}" || backports_yml_changed_exit=$? + if [[ "${backports_yml_changed_exit}" -eq 2 ]]; then + exit 1 + fi + if [[ "${backports_yml_changed_exit}" -ne 0 ]]; then echo ".backports.yml not changed, skipping backport ${label} trigger" exit 0 fi diff --git a/.buildkite/scripts/trigger_backport_lib.sh b/.buildkite/scripts/trigger_backport_lib.sh index d907b93e7fc..8cbf3515672 100644 --- a/.buildkite/scripts/trigger_backport_lib.sh +++ b/.buildkite/scripts/trigger_backport_lib.sh @@ -14,11 +14,15 @@ resolve_pr_number() { local commit="$1" local pr_number - pr_number=$(gh api "repos/elastic/integrations/commits/${commit}/pulls" --jq '.[0].number // empty' 2>/dev/null || true) + if ! pr_number="$(gh api "repos/elastic/integrations/commits/${commit}/pulls" \ + --jq '.[0].number // empty')"; then + echo "Failed to call GitHub API for commit ${commit}" >&2 + return 1 + fi if [[ -n "${pr_number}" ]]; then echo "Associated PR: #${pr_number}" >&2 else - echo "Could not resolve PR number for commit ${commit}, PR comments will be skipped" >&2 + echo "No PR found for commit ${commit}" >&2 fi echo "${pr_number}" } @@ -26,7 +30,10 @@ resolve_pr_number() { load_old_backports_inventory() { local git_ref="$1" local output_file="$2" - git show "${git_ref}:.backports.yml" > "${output_file}" 2>/dev/null + if ! git show "${git_ref}:.backports.yml" > "${output_file}" 2>/dev/null; then + echo "Note: .backports.yml not found at ref '${git_ref}'" >&2 + return 1 + fi } validate_backport_branch_name() { @@ -40,7 +47,12 @@ validate_backport_branch_name() { backports_yml_changed() { local from="$1" local to="$2" - git diff --name-only "${from}" "${to}" | grep -E '^\.backports\.yml$' > /dev/null + local changed_files + if ! changed_files="$(git diff --name-only "${from}" "${to}")"; then + echo "ERROR: git diff failed for refs '${from}' '${to}'" >&2 + return 2 + fi + grep -qE '^\.backports\.yml$' <<< "${changed_files}" } generate_trigger_pipeline() { From c811325d1879188e42dfa1ab19615a8cb3c8ada0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 12:21:48 +0200 Subject: [PATCH 12/23] [Backport pipeline] Add AddBackportEntry mage target Given a package name and base version, resolves the base commit via `dev/scripts/get_release_commit.sh`, inserts a new entry into `.backports.yml` in sorted order (package ascending, version descending), and prints the derived branch name. Co-Authored-By: Claude Sonnet 4.6 --- dev/backports/inventory.go | 124 +++++++++++++++++++++++++ dev/backports/inventory_test.go | 157 ++++++++++++++++++++++++++++++++ magefile.go | 18 ++++ 3 files changed, 299 insertions(+) diff --git a/dev/backports/inventory.go b/dev/backports/inventory.go index 9289af427ce..876c454dc88 100644 --- a/dev/backports/inventory.go +++ b/dev/backports/inventory.go @@ -5,6 +5,7 @@ package backports import ( + "bytes" "errors" "fmt" "os" @@ -111,6 +112,129 @@ func (e entry) activeResult(now time.Time) ActiveResult { return result } +// AddEntry inserts a new backport entry into the inventory at path. +// The branch name is derived as backport--.. +// archived is set to false and maintained_until to null. +// The entry is placed in sorted order: package name ascending, then version descending (newest first). +// Returns the derived branch name. +func AddEntry(path, packageName, baseVersion, baseCommit string) (string, error) { + v, err := semver.StrictNewVersion(baseVersion) + if err != nil { + return "", fmt.Errorf("invalid base_version %q: %w", baseVersion, err) + } + branch := fmt.Sprintf("backport-%s-%d.%d", packageName, v.Major(), v.Minor()) + + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading inventory: %w", err) + } + + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return "", fmt.Errorf("parsing inventory: %w", err) + } + + seq, err := backportsSequenceNode(&doc) + if err != nil { + return "", err + } + + pos := entryInsertPos(seq, packageName, v) + newNode := newEntryNode(packageName, branch, baseVersion, baseCommit) + + updated := make([]*yaml.Node, len(seq.Content)+1) + copy(updated, seq.Content[:pos]) + updated[pos] = newNode + copy(updated[pos+1:], seq.Content[pos:]) + seq.Content = updated + + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode(&doc); err != nil { + return "", fmt.Errorf("marshaling inventory: %w", err) + } + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + return "", fmt.Errorf("writing inventory: %w", err) + } + return branch, nil +} + +// backportsSequenceNode navigates from the document root to the sequence node +// under the "backports" key. +func backportsSequenceNode(doc *yaml.Node) (*yaml.Node, error) { + if doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 { + return nil, fmt.Errorf("unexpected document structure") + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + return nil, fmt.Errorf("expected mapping at document root") + } + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Value == "backports" { + seq := root.Content[i+1] + if seq.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("'backports' is not a sequence") + } + return seq, nil + } + } + return nil, fmt.Errorf("'backports' key not found in inventory") +} + +// entryInsertPos returns the index at which a new entry with the given package +// and version should be inserted to keep the sequence sorted (package ascending, +// then version descending within the same package — newest first). +func entryInsertPos(seq *yaml.Node, newPkg string, newVer *semver.Version) int { + for i, node := range seq.Content { + pkg := mappingFieldValue(node, "package") + if pkg > newPkg { + return i + } + if pkg == newPkg { + ver, err := semver.StrictNewVersion(mappingFieldValue(node, "base_version")) + if err == nil && ver.LessThan(newVer) { + return i + } + } + } + return len(seq.Content) +} + +// mappingFieldValue returns the scalar value for the given key in a YAML mapping node. +func mappingFieldValue(node *yaml.Node, key string) string { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return node.Content[i+1].Value + } + } + return "" +} + +// newEntryNode builds a YAML mapping node for a new backport entry. +// base_version and base_commit are double-quoted to match the existing file style. +func newEntryNode(pkg, branch, baseVersion, baseCommit string) *yaml.Node { + scalar := func(value, tag string, style yaml.Style) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value, Style: style} + } + key := func(k string) *yaml.Node { return scalar(k, "!!str", 0) } + str := func(v string) *yaml.Node { return scalar(v, "!!str", 0) } + quoted := func(v string) *yaml.Node { return scalar(v, "!!str", yaml.DoubleQuotedStyle) } + + return &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Content: []*yaml.Node{ + key("package"), str(pkg), + key("branch"), str(branch), + key("base_version"), quoted(baseVersion), + key("base_commit"), quoted(baseCommit), + key("maintained_until"), scalar("null", "!!null", 0), + key("archived"), scalar("false", "!!bool", 0), + }, + } +} + // ValidateInventory reads the .backports.yml inventory at path and returns a // combined error listing every schema violation found across all entries. // diff --git a/dev/backports/inventory_test.go b/dev/backports/inventory_test.go index 9ca5bfa11f8..eb0a21c76ea 100644 --- a/dev/backports/inventory_test.go +++ b/dev/backports/inventory_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) func writeTemp(t *testing.T, content string) string { @@ -629,4 +630,160 @@ func TestCheckActiveFileNotFound(t *testing.T) { assert.Contains(t, err.Error(), "reading inventory") } +func readInventory(t *testing.T, path string) inventory { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + var inv inventory + require.NoError(t, yaml.Unmarshal(data, &inv)) + return inv +} + +func TestAddEntry(t *testing.T) { + const twoEntries = `backports: + - package: aws + branch: backport-aws-3.0 + base_version: "3.0.0" + base_commit: "def5678901" + maintained_until: null + archived: false + - package: aws + branch: backport-aws-1.19 + base_version: "1.19.5" + base_commit: "abc1234567" + maintained_until: null + archived: false +` + + t.Run("inserts between versions of the same package", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "aws", "2.1.0", "aabbccddee") + require.NoError(t, err) + assert.Equal(t, "backport-aws-2.1", branch) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + assert.Equal(t, "backport-aws-3.0", inv.Backports[0].Branch) + assert.Equal(t, "backport-aws-2.1", inv.Backports[1].Branch) + assert.Equal(t, "backport-aws-1.19", inv.Backports[2].Branch) + }) + + t.Run("inserts between packages alphabetically", func(t *testing.T) { + inv := `backports: + - package: aws + branch: backport-aws-1.0 + base_version: "1.0.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false + - package: gcp + branch: backport-gcp-1.0 + base_version: "1.0.0" + base_commit: "ffeeddccbb" + maintained_until: null + archived: false +` + path := writeTemp(t, inv) + branch, err := AddEntry(path, "elastic_agent", "2.3.0", "112233aabb") + require.NoError(t, err) + assert.Equal(t, "backport-elastic_agent-2.3", branch) + + result := readInventory(t, path) + require.Len(t, result.Backports, 3) + assert.Equal(t, "backport-aws-1.0", result.Backports[0].Branch) + assert.Equal(t, "backport-elastic_agent-2.3", result.Backports[1].Branch) + assert.Equal(t, "backport-gcp-1.0", result.Backports[2].Branch) + }) + + t.Run("appends when new entry is last", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "zz_pkg", "1.0.0", "aabbccddee") + require.NoError(t, err) + assert.Equal(t, "backport-zz_pkg-1.0", branch) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + assert.Equal(t, "backport-zz_pkg-1.0", inv.Backports[2].Branch) + }) + + t.Run("prepends when new entry is first", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "aaa_pkg", "1.0.0", "aabbccddee") + require.NoError(t, err) + assert.Equal(t, "backport-aaa_pkg-1.0", branch) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + assert.Equal(t, "backport-aaa_pkg-1.0", inv.Backports[0].Branch) + assert.Equal(t, "backport-aws-3.0", inv.Backports[1].Branch) + }) + + t.Run("derives branch from major.minor only", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "aws", "2.5.3", "aabbccddee") + require.NoError(t, err) + assert.Equal(t, "backport-aws-2.5", branch) + }) + + t.Run("new entry has correct fields", func(t *testing.T) { + path := writeTemp(t, twoEntries) + _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee") + require.NoError(t, err) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + e := inv.Backports[1] + assert.Equal(t, "aws", e.Package) + assert.Equal(t, "backport-aws-2.1", e.Branch) + assert.Equal(t, "2.1.0", e.BaseVersion) + assert.Equal(t, "aabbccddee", e.BaseCommit) + assert.Nil(t, e.MaintainedUntil) + require.NotNil(t, e.Archived) + assert.False(t, *e.Archived) + }) + + t.Run("existing entries keep double-quoted style", func(t *testing.T) { + path := writeTemp(t, twoEntries) + _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee") + require.NoError(t, err) + + out, _ := os.ReadFile(path) + content := string(out) + assert.Contains(t, content, `base_version: "1.19.5"`) + assert.Contains(t, content, `base_version: "3.0.0"`) + }) + + t.Run("header comment is preserved", func(t *testing.T) { + inv := `# Backport inventory header +# +backports: + - package: aws + branch: backport-aws-1.0 + base_version: "1.0.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false +` + path := writeTemp(t, inv) + _, err := AddEntry(path, "aws", "2.0.0", "ffeeddccbb") + require.NoError(t, err) + + out, _ := os.ReadFile(path) + assert.Contains(t, string(out), "# Backport inventory header") + }) + + t.Run("invalid base_version returns error", func(t *testing.T) { + path := writeTemp(t, twoEntries) + _, err := AddEntry(path, "aws", "not-a-version", "aabbccddee") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid base_version") + }) + + t.Run("file not found returns error", func(t *testing.T) { + _, err := AddEntry("/no/such/file.yml", "aws", "1.0.0", "aabbccddee") + require.Error(t, err) + assert.Contains(t, err.Error(), "reading inventory") + }) +} + func ptr(s string) *string { return &s } diff --git a/magefile.go b/magefile.go index fa63677018e..236022befc8 100644 --- a/magefile.go +++ b/magefile.go @@ -324,6 +324,24 @@ func IsVersionLessThanLogsDBGA(version string) error { return nil } +// AddBackportEntry adds a new entry to .backports.yml for the given package and +// base version. The branch name is derived as backport--., +// archived is set to false, and maintained_until to null. The base commit is +// resolved via dev/scripts/get_release_commit.sh. The entry is inserted in +// sorted order (by package name ascending, then by version descending — newest first). +func AddBackportEntry(packageName, baseVersion string) error { + baseCommit, err := sh.Output("bash", "dev/scripts/get_release_commit.sh", "-p", packageName, "-v", baseVersion) + if err != nil { + return fmt.Errorf("resolving base commit for %s@%s: %w", packageName, baseVersion, err) + } + branch, err := backports.AddEntry(".backports.yml", packageName, baseVersion, strings.TrimSpace(baseCommit)) + if err != nil { + return err + } + fmt.Printf("Added: branch=%s base_commit=%s\n", branch, strings.TrimSpace(baseCommit)) + return nil +} + // CheckBackportBranchActive reports whether a backport branch is active per .backports.yml. // Prints ": active" or ": inactive ()". // Pass -json for JSON output: mage CheckBackportBranchActive -json From 2b7e11ff304258f25b86686de001705c6a265a5e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 13:24:20 +0200 Subject: [PATCH 13/23] [Backport pipeline] Apply code review fixes to trigger scripts and branch name validation Fixes: NOTIFY_STATUS unbound variable, redundant temp file cleanup, backports_yml_changed pathspec simplification, and branch name regex divergence between shell and Go (stricter regex, legacy exceptions, mage ValidateBackportsInventory replaces shell function). Co-Authored-By: Claude Sonnet 4.6 --- .buildkite/scripts/notify_backport_pr.sh | 3 ++ .buildkite/scripts/test_trigger_backport.sh | 18 ---------- .buildkite/scripts/trigger_backport.sh | 4 --- .buildkite/scripts/trigger_backport_lib.sh | 29 ++++++--------- dev/backports/inventory.go | 26 ++++++++++---- dev/backports/inventory_test.go | 39 ++++++++++++++++++++- 6 files changed, 71 insertions(+), 48 deletions(-) diff --git a/.buildkite/scripts/notify_backport_pr.sh b/.buildkite/scripts/notify_backport_pr.sh index 5a40e8bb456..8446fdd3eb4 100644 --- a/.buildkite/scripts/notify_backport_pr.sh +++ b/.buildkite/scripts/notify_backport_pr.sh @@ -14,6 +14,9 @@ PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default "${PACKAGE_ PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default "${PACKAGE_VERSION:-""}")" PR_NUMBER="$(buildkite-agent meta-data get PR_NUMBER --default "${PR_NUMBER:-""}")" +# Validate required env vars not available via meta-data. +: "${NOTIFY_STATUS:?NOTIFY_STATUS must be set to 'success' or 'failure'}" + if [[ -z "${PR_NUMBER}" ]]; then echo "PR_NUMBER not set, skipping PR notification" exit 0 diff --git a/.buildkite/scripts/test_trigger_backport.sh b/.buildkite/scripts/test_trigger_backport.sh index a462f187f24..5ee3206481a 100755 --- a/.buildkite/scripts/test_trigger_backport.sh +++ b/.buildkite/scripts/test_trigger_backport.sh @@ -192,24 +192,6 @@ generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || inv_exit=$? assert_exit_code "invalid old inventory: non-zero exit" "1" "${inv_exit}" -# --------------------------------------------------------------------------- -# Test: validate_backport_branch_name -# --------------------------------------------------------------------------- -echo "--- validate_backport_branch_name" - -# Valid names -assert_exit_code "valid: numeric minor" "0" "$(validate_backport_branch_name "backport-aws-1.19" 2>/dev/null; echo $?)" -assert_exit_code "valid: x minor" "0" "$(validate_backport_branch_name "backport-aws-6.x" 2>/dev/null; echo $?)" -assert_exit_code "valid: underscore in pkg" "0" "$(validate_backport_branch_name "backport-my_pkg-2.0" 2>/dev/null; echo $?)" -assert_exit_code "valid: single-digit minor" "0" "$(validate_backport_branch_name "backport-gcp-1.5" 2>/dev/null; echo $?)" - -# Invalid names -assert_exit_code "invalid: hyphen in pkg" "1" "$(validate_backport_branch_name "backport-my-pkg-2.0" 2>/dev/null; echo $?)" -assert_exit_code "invalid: missing minor" "1" "$(validate_backport_branch_name "backport-aws-1" 2>/dev/null; echo $?)" -assert_exit_code "invalid: mixed minor" "1" "$(validate_backport_branch_name "backport-aws-6.1x" 2>/dev/null; echo $?)" -assert_exit_code "invalid: wrong prefix" "1" "$(validate_backport_branch_name "not-backport-1.0" 2>/dev/null; echo $?)" -assert_exit_code "invalid: double-quote" "1" "$(validate_backport_branch_name 'backport-foo"bar-1.0' 2>/dev/null; echo $?)" - # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- diff --git a/.buildkite/scripts/trigger_backport.sh b/.buildkite/scripts/trigger_backport.sh index 3a8b1da92c1..07c960a649d 100755 --- a/.buildkite/scripts/trigger_backport.sh +++ b/.buildkite/scripts/trigger_backport.sh @@ -86,18 +86,14 @@ main() { generate_trigger_pipeline "${OLD_INVENTORY}" "${NEW_INVENTORY}" "${dry_run}" "${pr_number}" "${PIPELINE_FILE}" - rm -f "${OLD_INVENTORY}" - if [[ ! -s "${PIPELINE_FILE}" ]]; then echo "No new non-archived entries found, skipping ${label} trigger" - rm -f "${PIPELINE_FILE}" exit 0 fi echo "--- Uploading ${label} trigger(s)" cat "${PIPELINE_FILE}" buildkite-agent pipeline upload "${PIPELINE_FILE}" - rm -f "${PIPELINE_FILE}" } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then diff --git a/.buildkite/scripts/trigger_backport_lib.sh b/.buildkite/scripts/trigger_backport_lib.sh index 8cbf3515672..b23aa49ed29 100644 --- a/.buildkite/scripts/trigger_backport_lib.sh +++ b/.buildkite/scripts/trigger_backport_lib.sh @@ -7,9 +7,9 @@ # resolve_pr_number() looks up the PR that introduced a given commit via the GitHub API. # backports_yml_changed() checks whether .backports.yml was modified between two git refs. # load_old_backports_inventory() extracts .backports.yml at a given git ref into a file. -# generate_trigger_pipeline() iterates over a new .backports.yml, -# finds entries absent from the old inventory, and writes Buildkite trigger steps -# to a pipeline file. The caller decides whether to upload the result. +# generate_trigger_pipeline() validates the new inventory via mage ValidateBackportsInventory, +# iterates over a new .backports.yml, finds entries absent from the old inventory, and writes +# Buildkite trigger steps to a pipeline file. The caller decides whether to upload the result. resolve_pr_number() { local commit="$1" @@ -36,23 +36,15 @@ load_old_backports_inventory() { fi } -validate_backport_branch_name() { - local branch="$1" - if [[ ! "${branch}" =~ ^backport-[a-zA-Z0-9_]+-[0-9]+\.([0-9]+|x)$ ]]; then - echo "ERROR: invalid branch name '${branch}' — expected format: backport--." >&2 - return 1 - fi -} - backports_yml_changed() { local from="$1" local to="$2" - local changed_files - if ! changed_files="$(git diff --name-only "${from}" "${to}")"; then + local changed + if ! changed="$(git diff --name-only "${from}" "${to}" -- .backports.yml)"; then echo "ERROR: git diff failed for refs '${from}' '${to}'" >&2 return 2 fi - grep -qE '^\.backports\.yml$' <<< "${changed_files}" + [[ -n "${changed}" ]] } generate_trigger_pipeline() { @@ -72,6 +64,11 @@ generate_trigger_pipeline() { return 1 fi + if ! mage ValidateBackportsInventory; then + echo "ERROR: new inventory failed schema validation" >&2 + return 1 + fi + local label_prefix if [[ "${dry_run}" == "true" ]]; then label_prefix="Backport dry-run" @@ -80,10 +77,6 @@ generate_trigger_pipeline() { fi while IFS= read -r branch; do - if ! validate_backport_branch_name "${branch}"; then - return 1 - fi - local entry=".backports[] | select(.branch == \"${branch}\")" # Only trigger for entries that are new (absent from the old inventory). diff --git a/dev/backports/inventory.go b/dev/backports/inventory.go index 876c454dc88..153af6daebf 100644 --- a/dev/backports/inventory.go +++ b/dev/backports/inventory.go @@ -48,14 +48,26 @@ var duplicatePackageVersionExceptions = map[string]struct{}{ // branchRE matches a valid backport branch name: // -// backport-- +// backport--. // // where is one or more letters, digits, or underscores, and -// starts with a digit followed by digits, dots, and an optional -// trailing 'x' wildcard (e.g. "6.x" or "6.14.x"). +// is exactly . or .x (e.g. "3.17" or "6.x"). // Whitespace, quotes, colons, semicolons, and all other special characters // are not permitted. -var branchRE = regexp.MustCompile(`^backport-[a-zA-Z0-9_]+-[0-9][0-9.]*x?$`) +var branchRE = regexp.MustCompile(`^backport-[a-zA-Z0-9_]+-[0-9]+\.([0-9]+|x)$`) + +// legacyBranchNames lists branch names that predate the current naming convention +// and are intentionally exempt from the branchRE format check. +// Each entry must include a comment explaining why the exception exists. +var legacyBranchNames = map[string]struct{}{ + // backport-aws-7.15.0 used a three-component version before the + // backport--. convention was established. + "backport-aws-7.15.0": {}, + // backport-security_detection_engine-8.9.10 was a one-off exception to + // publish a missing package version; it used a patch-level version rather + // than the standard major.minor format. + "backport-security_detection_engine-8.9.10": {}, +} // ActiveResult is the result of a CheckActive call. type ActiveResult struct { @@ -286,9 +298,9 @@ func validateEntryFields(i int, e entry, knownPackages map[string]struct{}, pack if e.Branch == "" { errs = append(errs, fmt.Errorf("%s: missing required field 'branch'", id)) - } else if !branchRE.MatchString(e.Branch) { - errs = append(errs, fmt.Errorf("%s: invalid branch %q: must match backport-- "+ - "(letters/digits/underscores in package name, version starts with a digit; "+ + } else if _, isLegacy := legacyBranchNames[e.Branch]; !isLegacy && !branchRE.MatchString(e.Branch) { + errs = append(errs, fmt.Errorf("%s: invalid branch %q: must match backport--. "+ + "(letters/digits/underscores in package name; "+ "no whitespace, quotes, colons, semicolons or other special characters)", id, e.Branch)) } diff --git a/dev/backports/inventory_test.go b/dev/backports/inventory_test.go index eb0a21c76ea..d612e3d44be 100644 --- a/dev/backports/inventory_test.go +++ b/dev/backports/inventory_test.go @@ -221,7 +221,7 @@ func TestValidateInventory(t *testing.T) { `, }, { - title: "valid branch with x wildcard and minor", + title: "invalid branch — three-component version (major.minor.x)", contents: `backports: - package: aws branch: backport-aws-6.14.x @@ -229,6 +229,43 @@ func TestValidateInventory(t *testing.T) { base_commit: "5b593f6681" maintained_until: null archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid branch — three-component version (major.minor.patch)", + contents: `backports: + - package: aws + branch: backport-aws-1.2.3 + base_version: "1.2.3" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "legacy branch backport-aws-7.15.0 passes despite three-component version", + contents: `backports: + - package: aws + branch: backport-aws-7.15.0 + base_version: "7.15.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + }, + { + title: "legacy branch backport-security_detection_engine-8.9.10 passes despite three-component version", + contents: `backports: + - package: security_detection_engine + branch: backport-security_detection_engine-8.9.10 + base_version: "8.9.10" + base_commit: "5b593f6681" + maintained_until: null + archived: false `, }, { From c4e95059b00ab978e9fc325afc03cd6cb9a350b7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 17:14:59 +0200 Subject: [PATCH 14/23] [Backport pipeline] Validate package name in AddEntry and split per-target mage mock in tests Co-authored-by: Claude Sonnet 4.6 --- .buildkite/scripts/test_trigger_backport.sh | 44 ++++++++++++++++----- dev/backports/inventory.go | 14 ++++++- dev/backports/inventory_test.go | 20 +++++----- magefile.go | 5 ++- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/.buildkite/scripts/test_trigger_backport.sh b/.buildkite/scripts/test_trigger_backport.sh index 5ee3206481a..ecfc9d4ac3d 100755 --- a/.buildkite/scripts/test_trigger_backport.sh +++ b/.buildkite/scripts/test_trigger_backport.sh @@ -15,15 +15,20 @@ pass=0 fail=0 # --------------------------------------------------------------------------- -# Mock: mage — set MOCK_MAGE_EXIT to control CheckBackportBranchActive result. -# 0 = active (default) -# 1 = inactive -# 2 = error +# Mock: mage — per-target exit code control. +# MOCK_MAGE_VALIDATE_EXIT controls ValidateBackportsInventory (default 0 = pass) +# MOCK_MAGE_CHECK_EXIT controls CheckBackportBranchActive (default 0 = active, +# 1 = inactive, +# 2 = error) # --------------------------------------------------------------------------- mage() { - return "${MOCK_MAGE_EXIT:-0}" + case "$1" in + ValidateBackportsInventory) return "${MOCK_MAGE_VALIDATE_EXIT:-0}" ;; + CheckBackportBranchActive) return "${MOCK_MAGE_CHECK_EXIT:-0}" ;; + esac } -MOCK_MAGE_EXIT=0 +MOCK_MAGE_VALIDATE_EXIT=0 +MOCK_MAGE_CHECK_EXIT=0 # --------------------------------------------------------------------------- # Helpers @@ -135,9 +140,9 @@ write_inventory "${NEW}" \ "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ "backport-aws-2.0" "aws" "2.0.0" "def456" "true" -MOCK_MAGE_EXIT=1 +MOCK_MAGE_CHECK_EXIT=1 run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" -MOCK_MAGE_EXIT=0 +MOCK_MAGE_CHECK_EXIT=0 assert_equals "inactive entry: no steps generated" "" "$(cat "${OUT}")" @@ -152,14 +157,33 @@ write_inventory "${NEW}" \ "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ "backport-aws-2.0" "aws" "2.0.0" "def456" "false" -MOCK_MAGE_EXIT=2 +MOCK_MAGE_CHECK_EXIT=2 err_exit=0 : > "${OUT}" generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || err_exit=$? -MOCK_MAGE_EXIT=0 +MOCK_MAGE_CHECK_EXIT=0 assert_exit_code "mage error: non-zero exit returned" "1" "${err_exit}" +# --------------------------------------------------------------------------- +# Test: ValidateBackportsInventory failure causes function to return non-zero +# --------------------------------------------------------------------------- +echo "--- validate inventory failure returns non-zero" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +MOCK_MAGE_VALIDATE_EXIT=1 +val_exit=0 +: > "${OUT}" +generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || val_exit=$? +MOCK_MAGE_VALIDATE_EXIT=0 + +assert_exit_code "validate failure: non-zero exit returned" "1" "${val_exit}" + # --------------------------------------------------------------------------- # Test: multiple new entries all appear in pipeline # --------------------------------------------------------------------------- diff --git a/dev/backports/inventory.go b/dev/backports/inventory.go index 153af6daebf..cd089a7fea4 100644 --- a/dev/backports/inventory.go +++ b/dev/backports/inventory.go @@ -128,14 +128,26 @@ func (e entry) activeResult(now time.Time) ActiveResult { // The branch name is derived as backport--.. // archived is set to false and maintained_until to null. // The entry is placed in sorted order: package name ascending, then version descending (newest first). +// packagesDir is the path to the packages/ directory used to verify that packageName names a real +// package. Pass an empty string to skip this check. // Returns the derived branch name. -func AddEntry(path, packageName, baseVersion, baseCommit string) (string, error) { +func AddEntry(path, packageName, baseVersion, baseCommit, packagesDir string) (string, error) { v, err := semver.StrictNewVersion(baseVersion) if err != nil { return "", fmt.Errorf("invalid base_version %q: %w", baseVersion, err) } branch := fmt.Sprintf("backport-%s-%d.%d", packageName, v.Major(), v.Minor()) + knownPackages, err := buildKnownPackages(packagesDir) + if err != nil { + return "", fmt.Errorf("loading packages from %s: %w", packagesDir, err) + } + if knownPackages != nil { + if _, ok := knownPackages[packageName]; !ok { + return "", fmt.Errorf("unknown package %q: not found under %s", packageName, packagesDir) + } + } + data, err := os.ReadFile(path) if err != nil { return "", fmt.Errorf("reading inventory: %w", err) diff --git a/dev/backports/inventory_test.go b/dev/backports/inventory_test.go index d612e3d44be..59f9052fc62 100644 --- a/dev/backports/inventory_test.go +++ b/dev/backports/inventory_test.go @@ -694,7 +694,7 @@ func TestAddEntry(t *testing.T) { t.Run("inserts between versions of the same package", func(t *testing.T) { path := writeTemp(t, twoEntries) - branch, err := AddEntry(path, "aws", "2.1.0", "aabbccddee") + branch, err := AddEntry(path, "aws", "2.1.0", "aabbccddee", "") require.NoError(t, err) assert.Equal(t, "backport-aws-2.1", branch) @@ -721,7 +721,7 @@ func TestAddEntry(t *testing.T) { archived: false ` path := writeTemp(t, inv) - branch, err := AddEntry(path, "elastic_agent", "2.3.0", "112233aabb") + branch, err := AddEntry(path, "elastic_agent", "2.3.0", "112233aabb", "") require.NoError(t, err) assert.Equal(t, "backport-elastic_agent-2.3", branch) @@ -734,7 +734,7 @@ func TestAddEntry(t *testing.T) { t.Run("appends when new entry is last", func(t *testing.T) { path := writeTemp(t, twoEntries) - branch, err := AddEntry(path, "zz_pkg", "1.0.0", "aabbccddee") + branch, err := AddEntry(path, "zz_pkg", "1.0.0", "aabbccddee", "") require.NoError(t, err) assert.Equal(t, "backport-zz_pkg-1.0", branch) @@ -745,7 +745,7 @@ func TestAddEntry(t *testing.T) { t.Run("prepends when new entry is first", func(t *testing.T) { path := writeTemp(t, twoEntries) - branch, err := AddEntry(path, "aaa_pkg", "1.0.0", "aabbccddee") + branch, err := AddEntry(path, "aaa_pkg", "1.0.0", "aabbccddee", "") require.NoError(t, err) assert.Equal(t, "backport-aaa_pkg-1.0", branch) @@ -757,14 +757,14 @@ func TestAddEntry(t *testing.T) { t.Run("derives branch from major.minor only", func(t *testing.T) { path := writeTemp(t, twoEntries) - branch, err := AddEntry(path, "aws", "2.5.3", "aabbccddee") + branch, err := AddEntry(path, "aws", "2.5.3", "aabbccddee", "") require.NoError(t, err) assert.Equal(t, "backport-aws-2.5", branch) }) t.Run("new entry has correct fields", func(t *testing.T) { path := writeTemp(t, twoEntries) - _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee") + _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee", "") require.NoError(t, err) inv := readInventory(t, path) @@ -781,7 +781,7 @@ func TestAddEntry(t *testing.T) { t.Run("existing entries keep double-quoted style", func(t *testing.T) { path := writeTemp(t, twoEntries) - _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee") + _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee", "") require.NoError(t, err) out, _ := os.ReadFile(path) @@ -802,7 +802,7 @@ backports: archived: false ` path := writeTemp(t, inv) - _, err := AddEntry(path, "aws", "2.0.0", "ffeeddccbb") + _, err := AddEntry(path, "aws", "2.0.0", "ffeeddccbb", "") require.NoError(t, err) out, _ := os.ReadFile(path) @@ -811,13 +811,13 @@ backports: t.Run("invalid base_version returns error", func(t *testing.T) { path := writeTemp(t, twoEntries) - _, err := AddEntry(path, "aws", "not-a-version", "aabbccddee") + _, err := AddEntry(path, "aws", "not-a-version", "aabbccddee", "") require.Error(t, err) assert.Contains(t, err.Error(), "invalid base_version") }) t.Run("file not found returns error", func(t *testing.T) { - _, err := AddEntry("/no/such/file.yml", "aws", "1.0.0", "aabbccddee") + _, err := AddEntry("/no/such/file.yml", "aws", "1.0.0", "aabbccddee", "") require.Error(t, err) assert.Contains(t, err.Error(), "reading inventory") }) diff --git a/magefile.go b/magefile.go index 236022befc8..19fc3957b2b 100644 --- a/magefile.go +++ b/magefile.go @@ -334,11 +334,12 @@ func AddBackportEntry(packageName, baseVersion string) error { if err != nil { return fmt.Errorf("resolving base commit for %s@%s: %w", packageName, baseVersion, err) } - branch, err := backports.AddEntry(".backports.yml", packageName, baseVersion, strings.TrimSpace(baseCommit)) + commit := strings.TrimSpace(baseCommit) + branch, err := backports.AddEntry(".backports.yml", packageName, baseVersion, commit, "packages") if err != nil { return err } - fmt.Printf("Added: branch=%s base_commit=%s\n", branch, strings.TrimSpace(baseCommit)) + fmt.Printf("Added: branch=%s base_commit=%s\n", branch, commit) return nil } From 16c9ddc5f47d01de5191f21f99755b5793aa7ae8 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 17:22:32 +0200 Subject: [PATCH 15/23] Test a subset of packages - to be removed --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 94bfb8adb7c..e45e20475f4 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -925,7 +925,7 @@ teardown_test_package() { # list all directories that are packages from the root of the repository list_all_directories() { - mage -d "${WORKSPACE}" listPackages + mage -d "${WORKSPACE}" listPackages |grep -E "^packages/elastic_package_registry$" } check_package() { From f3f16c8201440f4562bfde4b02e80c03a340173e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 17:35:51 +0200 Subject: [PATCH 16/23] Revert change in backport_branch.sh --- .buildkite/scripts/backport_branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/backport_branch.sh b/.buildkite/scripts/backport_branch.sh index 6fe245c96e5..3248c90f093 100755 --- a/.buildkite/scripts/backport_branch.sh +++ b/.buildkite/scripts/backport_branch.sh @@ -342,7 +342,7 @@ fi echo "--- Creating the branch: $BACKPORT_BRANCH_NAME from the commit: $BASE_COMMIT" createLocalBackportBranch "$BACKPORT_BRANCH_NAME" "$BASE_COMMIT" -MSG="The backport branch: **$BACKPORT_BRANCH_NAME** has been created" +MSG="The backport branch: **$BACKPORT_BRANCH_NAME** has been created." echo "+++ Adding CI files into the branch ${BACKPORT_BRANCH_NAME}" updateBackportBranchContents From 53124f8eb9771a3ac091d5c797eaf0ecb3e9b4c6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 17:58:24 +0200 Subject: [PATCH 17/23] Update condition for main branch --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7cb180bed27..8115069c157 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -110,7 +110,7 @@ steps: - ".backports.yml" if: | build.env('BUILDKITE_PULL_REQUEST') == "false" && - build.env('BUILDKITE_BRANCH') == "main" && + build.branch == "main" && build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" - label: ":junit: Sources Junit annotate" From 16e76f6d0f6ee9da74fd9d9a0c137e47d7f16b12 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 18:05:37 +0200 Subject: [PATCH 18/23] Update condition about teams in pipeline.backport.yml --- .buildkite/pipeline.backport.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 88860d52bec..11f77297616 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -14,11 +14,11 @@ steps: - label: "Check that it runs from UI" key: "check-ui" command: - - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG pipeline can only be triggered from the UI by members of the 'ecosystem' team, or via a trigger step from the 'integrations' pipeline.\" --style 'warning'" + - "buildkite-agent annotate \"The $${BUILDKITE_PIPELINE_SLUG} pipeline can only be triggered from the UI by members of the 'ecosystem' team, or via a trigger step from the 'integrations' pipeline.\" --style 'warning'" - "exit 1" if: | !( - (build.source == 'ui' && build.env('BUILDKITE_BUILD_CREATOR_TEAMS') =~ /(^|:)ecosystem(:|$)/) || + (build.source == 'ui' && build.creator.teams includes "ecosystem") || (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') ) From 3930846f130cb0233c222e6ec495655019d02ba6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 18:14:34 +0200 Subject: [PATCH 19/23] [Backport pipeline] Install yq before running trigger backport tests in CI Co-authored-by: Claude Sonnet 4.6 --- .buildkite/scripts/run_buildkite_scripts_tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/scripts/run_buildkite_scripts_tests.sh b/.buildkite/scripts/run_buildkite_scripts_tests.sh index 948b41ecb09..fe6eb2a16ac 100755 --- a/.buildkite/scripts/run_buildkite_scripts_tests.sh +++ b/.buildkite/scripts/run_buildkite_scripts_tests.sh @@ -47,4 +47,10 @@ run_tests_if_exists "${REPO_ROOT}/.buildkite/scripts/test_check_changelog_entrie echo "" echo "=== Running trigger_backport_lib.sh tests ===" +# yq is required by generate_trigger_pipeline(); install it if not already available. +if ! command -v yq &>/dev/null; then + source "${REPO_ROOT}/.buildkite/scripts/common.sh" + add_bin_path + with_yq +fi run_tests_if_exists "${REPO_ROOT}/.buildkite/scripts/test_trigger_backport.sh" From a8135016e4ef633e05ddb40818d11976ca6569ca Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 18:42:39 +0200 Subject: [PATCH 20/23] Skip trigger the integrations tests - to be reverted --- .buildkite/pipeline.yml | 2 ++ .buildkite/scripts/common.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8115069c157..640076dbd9c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -162,6 +162,8 @@ steps: allow_failure: false - step: "check-buildkite-scripts" allow_failure: false + if: | + build.branch == "nonexisting" - wait: ~ continue_on_failure: true diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e45e20475f4..94bfb8adb7c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -925,7 +925,7 @@ teardown_test_package() { # list all directories that are packages from the root of the repository list_all_directories() { - mage -d "${WORKSPACE}" listPackages |grep -E "^packages/elastic_package_registry$" + mage -d "${WORKSPACE}" listPackages } check_package() { From 6deb3e8271501d9541850ea8252c12ee192da4b0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 18:50:38 +0200 Subject: [PATCH 21/23] Update permission for notify_backport_pr script --- .buildkite/scripts/notify_backport_pr.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .buildkite/scripts/notify_backport_pr.sh diff --git a/.buildkite/scripts/notify_backport_pr.sh b/.buildkite/scripts/notify_backport_pr.sh old mode 100644 new mode 100755 From c8f110a1869e7a494350a4f16eea58c3b72ac7ec Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 18:57:28 +0200 Subject: [PATCH 22/23] Add sections in the output of notify backport pr --- .buildkite/scripts/notify_backport_pr.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/scripts/notify_backport_pr.sh b/.buildkite/scripts/notify_backport_pr.sh index 8446fdd3eb4..73f3da49478 100755 --- a/.buildkite/scripts/notify_backport_pr.sh +++ b/.buildkite/scripts/notify_backport_pr.sh @@ -15,6 +15,7 @@ PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default "${PA PR_NUMBER="$(buildkite-agent meta-data get PR_NUMBER --default "${PR_NUMBER:-""}")" # Validate required env vars not available via meta-data. +echo "--- Validating required env vars" : "${NOTIFY_STATUS:?NOTIFY_STATUS must be set to 'success' or 'failure'}" if [[ -z "${PR_NUMBER}" ]]; then @@ -22,9 +23,11 @@ if [[ -z "${PR_NUMBER}" ]]; then exit 0 fi +echo "--- Adding GitHub CLI to PATH" add_bin_path with_github_cli +echo "--- Creating body file" BODY_FILE="$(mktemp)" trap 'rm -f "${BODY_FILE}"' EXIT @@ -42,5 +45,6 @@ Check the [Buildkite build](${BUILDKITE_BUILD_URL}) for details. EOF fi +echo "--- Creating new GitHub PR comment" RUN_ID="backport-${BACKPORT_BRANCH_NAME}-${BUILDKITE_BUILD_NUMBER:-0}-${BUILDKITE_RETRY_COUNT:-0}" retry 3 create_new_gh_pr_comment "elastic" "integrations" "${PR_NUMBER}" "${RUN_ID}" "${BODY_FILE}" From 89f7d03be40d79e4dcf6ef22d20367134b50e738 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jun 2026 19:16:29 +0200 Subject: [PATCH 23/23] Test other team to fail the safety guard check --- .buildkite/pipeline.backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 11f77297616..341ab6680ac 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -18,7 +18,7 @@ steps: - "exit 1" if: | !( - (build.source == 'ui' && build.creator.teams includes "ecosystem") || + (build.source == 'ui' && build.creator.teams includes "other") || (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') )