diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 0f37b461cac..341ab6680ac 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -5,6 +5,7 @@ name: "integrations-backport" env: SETUP_GVM_VERSION: "v0.6.0" YQ_VERSION: 'v4.35.2' + GH_CLI_VERSION: "2.29.0" # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" @@ -13,11 +14,11 @@ steps: - label: "Check that it runs from UI" key: "check-ui" command: - - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG is used only for running from UI or a trigger step!\" --style 'warning'" + - "buildkite-agent annotate \"The $${BUILDKITE_PIPELINE_SLUG} pipeline can only be triggered from the UI by members of the 'ecosystem' team, or via a trigger step from the 'integrations' pipeline.\" --style 'warning'" - "exit 1" if: | !( - build.source == 'ui' || + (build.source == 'ui' && build.creator.teams includes "other") || (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') ) @@ -70,3 +71,20 @@ steps: depends_on: - step: "input-variables" allow_failure: false + + - label: ":github: Notify PR" + key: "notify-pr" + command: | + outcome=$$(buildkite-agent step get "outcome" --step "create-backport-branch") + if [[ "$${outcome}" == "passed" ]]; then + NOTIFY_STATUS=success .buildkite/scripts/notify_backport_pr.sh + else + NOTIFY_STATUS=failure .buildkite/scripts/notify_backport_pr.sh + fi + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + depends_on: + - step: "create-backport-branch" + allow_failure: true diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 01bcc6d2967..640076dbd9c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -86,7 +86,7 @@ steps: - label: ":git: Trigger backport dry-runs" key: "trigger-backport-dryrun" - command: ".buildkite/scripts/trigger_backport_dryrun.sh" + command: ".buildkite/scripts/trigger_backport.sh" agents: image: "${LINUX_AGENT_IMAGE}" depends_on: @@ -99,6 +99,20 @@ steps: build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" && build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == "main" + - label: ":git: Create backport branches for new entries" + key: "trigger-backport-create" + command: ".buildkite/scripts/trigger_backport.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + plugins: + - elastic/vault-github-token#v0.1.0: + if_changed: + - ".backports.yml" + if: | + build.env('BUILDKITE_PULL_REQUEST') == "false" && + build.branch == "main" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" + - label: ":junit: Sources Junit annotate" agents: # requires at least "bash", "curl" and "git" @@ -148,6 +162,8 @@ steps: allow_failure: false - step: "check-buildkite-scripts" allow_failure: false + if: | + build.branch == "nonexisting" - wait: ~ continue_on_failure: true diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index a46f45b5172..1adc1790780 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -23,6 +23,7 @@ "^.buildkite/pull-requests.json$", "^.buildkite/scripts/backport_branch.sh$", "^.buildkite/scripts/build_packages.sh$", + "^.buildkite/scripts/notify_backport_pr.sh$", "^.github/dependabot.yml$", "^.github/workflows/", "^.github/stale.yml$", diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7a7640ad7d8..94bfb8adb7c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -789,7 +789,9 @@ is_pr_affected() { '\.buildkite/pull-requests\.json' '\.buildkite/scripts/backport_branch\.sh' '\.buildkite/scripts/check_backports_inventory\.sh' - '\.buildkite/scripts/trigger_backport_dryrun\.sh' + '\.buildkite/scripts/notify_backport_pr\.sh' + '\.buildkite/scripts/trigger_backport\.sh' + '\.buildkite/scripts/trigger_backport_lib\.sh' '\.buildkite/scripts/build_packages\.sh' '\.buildkite/scripts/check_changelog_entries\.sh' '\.buildkite/scripts/packages/.+\.sh' @@ -798,6 +800,7 @@ is_pr_affected() { '\.buildkite/scripts/run_dev_scripts_tests\.sh' '\.buildkite/scripts/test_check_changelog_entries\.sh' '\.buildkite/scripts/test_helpers\.sh' + '\.buildkite/scripts/test_trigger_backport\.sh' '\.github/dependabot\.yml' '\.github/stale\.yml' '\.github/workflows/' @@ -1242,6 +1245,36 @@ delete_and_create_gh_pr_comment() { --body "${contents}" } +# Posts a new comment on every pipeline run/retry while staying idempotent +# within a single attempt. Pass a unique id per attempt (e.g. build-number + +# retry-count) — if a comment with that id already exists the call is a no-op, +# so transient gh failures can be retried safely without double-posting. +create_new_gh_pr_comment() { + local owner="$1" + local repo="$2" + local pr_number="$3" + local id="$4" + local comment_file="$5" + local metadata="" + + local comment_id + comment_id=$(get_comment_with_pattern "${owner}" "${repo}" "${pr_number}" "${metadata}") + if [[ -n "${comment_id}" ]]; then + echo "Comment already posted for id=${id}, skipping" + return + fi + + local contents + contents="$(cat "${comment_file}")" + printf -v contents '%s\n%s' "${contents}" "${metadata}" + + echo "Creating new comment" + gh pr comment \ + "${pr_number}" \ + --repo "${owner}/${repo}" \ + --body "${contents}" +} + # FIXME: In a Pull Request that there are more than 100 comments, # if the comment is older than those 100 comments, it won't be found due to pagination get_comment_with_pattern() { diff --git a/.buildkite/scripts/notify_backport_pr.sh b/.buildkite/scripts/notify_backport_pr.sh new file mode 100755 index 00000000000..73f3da49478 --- /dev/null +++ b/.buildkite/scripts/notify_backport_pr.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Posts a comment on the PR that triggered the backport branch creation, +# reporting success or failure of the branch creation step. +# Expects env vars: PR_NUMBER, NOTIFY_STATUS (success|failure), +# BACKPORT_BRANCH_NAME, PACKAGE_NAME, PACKAGE_VERSION, +# BUILDKITE_BUILD_URL + +source .buildkite/scripts/common.sh + +set -euo pipefail + +BACKPORT_BRANCH_NAME="$(buildkite-agent meta-data get BACKPORT_BRANCH_NAME --default "${BACKPORT_BRANCH_NAME:-""}")" +PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default "${PACKAGE_NAME:-""}")" +PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default "${PACKAGE_VERSION:-""}")" +PR_NUMBER="$(buildkite-agent meta-data get PR_NUMBER --default "${PR_NUMBER:-""}")" + +# Validate required env vars not available via meta-data. +echo "--- Validating required env vars" +: "${NOTIFY_STATUS:?NOTIFY_STATUS must be set to 'success' or 'failure'}" + +if [[ -z "${PR_NUMBER}" ]]; then + echo "PR_NUMBER not set, skipping PR notification" + exit 0 +fi + +echo "--- Adding GitHub CLI to PATH" +add_bin_path +with_github_cli + +echo "--- Creating body file" +BODY_FILE="$(mktemp)" +trap 'rm -f "${BODY_FILE}"' EXIT + +if [[ "${NOTIFY_STATUS}" == "success" ]]; then + cat > "${BODY_FILE}" < "${BODY_FILE}" </dev/null; then + source "${REPO_ROOT}/.buildkite/scripts/common.sh" + add_bin_path + with_yq +fi +run_tests_if_exists "${REPO_ROOT}/.buildkite/scripts/test_trigger_backport.sh" diff --git a/.buildkite/scripts/test_helpers.sh b/.buildkite/scripts/test_helpers.sh index a31b51e452a..b1e0a49764e 100644 --- a/.buildkite/scripts/test_helpers.sh +++ b/.buildkite/scripts/test_helpers.sh @@ -27,3 +27,29 @@ assert_exit_code() { (( fail++ )) || true fi } + +assert_file_contains() { + local description="$1" + local needle="$2" + local file="$3" + if grep -qF "${needle}" "${file}" 2>/dev/null; then + echo "PASS: ${description}" + (( pass++ )) || true + else + echo "FAIL: ${description} — '${needle}' not found in ${file}" + (( fail++ )) || true + fi +} + +assert_file_not_contains() { + local description="$1" + local needle="$2" + local file="$3" + if ! grep -qF "${needle}" "${file}" 2>/dev/null; then + echo "PASS: ${description}" + (( pass++ )) || true + else + echo "FAIL: ${description} — '${needle}' unexpectedly found in ${file}" + (( fail++ )) || true + fi +} diff --git a/.buildkite/scripts/test_trigger_backport.sh b/.buildkite/scripts/test_trigger_backport.sh new file mode 100755 index 00000000000..ecfc9d4ac3d --- /dev/null +++ b/.buildkite/scripts/test_trigger_backport.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# Unit tests for generate_trigger_pipeline() from trigger_backport_lib.sh. +# Run directly or via .buildkite/scripts/run_buildkite_scripts_tests.sh. +# +# Requires: yq (installed via with_yq in CI; must be available locally too). + +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" + +source "${REPO_ROOT}/.buildkite/scripts/trigger_backport_lib.sh" +source "${REPO_ROOT}/.buildkite/scripts/test_helpers.sh" + +pass=0 +fail=0 + +# --------------------------------------------------------------------------- +# Mock: mage — per-target exit code control. +# MOCK_MAGE_VALIDATE_EXIT controls ValidateBackportsInventory (default 0 = pass) +# MOCK_MAGE_CHECK_EXIT controls CheckBackportBranchActive (default 0 = active, +# 1 = inactive, +# 2 = error) +# --------------------------------------------------------------------------- +mage() { + case "$1" in + ValidateBackportsInventory) return "${MOCK_MAGE_VALIDATE_EXIT:-0}" ;; + CheckBackportBranchActive) return "${MOCK_MAGE_CHECK_EXIT:-0}" ;; + esac +} +MOCK_MAGE_VALIDATE_EXIT=0 +MOCK_MAGE_CHECK_EXIT=0 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +WORK_DIR="" +cleanup() { + [[ -n "${WORK_DIR}" ]] && rm -rf "${WORK_DIR}" +} +trap cleanup EXIT +WORK_DIR="$(mktemp -d)" + +# Write a .backports.yml with one or more entries. +# Each entry requires exactly 5 args: branch pkg base_version base_commit archived +# Usage: write_inventory ... +write_inventory() { + local file="$1"; shift + printf 'backports:\n' > "${file}" + while [[ $# -ge 5 ]]; do + local branch="$1" pkg="$2" base_version="$3" base_commit="$4" archived="$5" + cat >> "${file}" < +run_generate() { + local old="$1" new="$2" dry_run="$3" pr_number="$4" out="$5" + : > "${out}" + generate_trigger_pipeline "${old}" "${new}" "${dry_run}" "${pr_number}" "${out}" || true +} + +OLD="${WORK_DIR}/old.yml" +NEW="${WORK_DIR}/new.yml" +OUT="${WORK_DIR}/pipeline.yml" + +# --------------------------------------------------------------------------- +# Test: new entry in dry-run mode +# --------------------------------------------------------------------------- +echo "--- new entry — dry-run mode" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" + +assert_file_contains "dry-run: step header present" "steps:" "${OUT}" +assert_file_contains "dry-run: label is Backport dry-run" 'Backport dry-run: backport-aws-2.0' "${OUT}" +assert_file_contains "dry-run: DRY_RUN is true" 'DRY_RUN: "true"' "${OUT}" +assert_file_contains "dry-run: PACKAGE_NAME correct" 'PACKAGE_NAME: "aws"' "${OUT}" +assert_file_contains "dry-run: PACKAGE_VERSION correct" 'PACKAGE_VERSION: "2.0.0"' "${OUT}" +assert_file_contains "dry-run: BASE_COMMIT correct" 'BASE_COMMIT: "def456"' "${OUT}" +assert_file_contains "dry-run: BACKPORT_BRANCH_NAME correct" 'BACKPORT_BRANCH_NAME: "backport-aws-2.0"' "${OUT}" +assert_file_not_contains "dry-run: PR_NUMBER absent when empty" "PR_NUMBER" "${OUT}" + +# --------------------------------------------------------------------------- +# Test: new entry in create mode +# --------------------------------------------------------------------------- +echo "--- new entry — create mode" + +run_generate "${OLD}" "${NEW}" "false" "" "${OUT}" + +assert_file_contains "create: label is Backport create" 'Backport create: backport-aws-2.0' "${OUT}" +assert_file_contains "create: DRY_RUN is false" 'DRY_RUN: "false"' "${OUT}" + +# --------------------------------------------------------------------------- +# Test: PR_NUMBER included when provided, omitted when empty +# --------------------------------------------------------------------------- +echo "--- PR_NUMBER" + +run_generate "${OLD}" "${NEW}" "false" "9999" "${OUT}" +assert_file_contains "PR_NUMBER included when set" 'PR_NUMBER: "9999"' "${OUT}" + +run_generate "${OLD}" "${NEW}" "false" "" "${OUT}" +assert_file_not_contains "PR_NUMBER omitted when empty" "PR_NUMBER" "${OUT}" + +# --------------------------------------------------------------------------- +# Test: existing entry is skipped +# --------------------------------------------------------------------------- +echo "--- existing entry skipped" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" + +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" +assert_equals "existing entry: no steps generated" "" "$(cat "${OUT}")" + +# --------------------------------------------------------------------------- +# Test: inactive entry (mage returns 1) is skipped +# --------------------------------------------------------------------------- +echo "--- inactive entry skipped" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "true" + +MOCK_MAGE_CHECK_EXIT=1 +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" +MOCK_MAGE_CHECK_EXIT=0 + +assert_equals "inactive entry: no steps generated" "" "$(cat "${OUT}")" + +# --------------------------------------------------------------------------- +# Test: mage error (exit code 2) causes function to return non-zero +# --------------------------------------------------------------------------- +echo "--- mage error propagated" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +MOCK_MAGE_CHECK_EXIT=2 +err_exit=0 +: > "${OUT}" +generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || err_exit=$? +MOCK_MAGE_CHECK_EXIT=0 + +assert_exit_code "mage error: non-zero exit returned" "1" "${err_exit}" + +# --------------------------------------------------------------------------- +# Test: ValidateBackportsInventory failure causes function to return non-zero +# --------------------------------------------------------------------------- +echo "--- validate inventory failure returns non-zero" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +MOCK_MAGE_VALIDATE_EXIT=1 +val_exit=0 +: > "${OUT}" +generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || val_exit=$? +MOCK_MAGE_VALIDATE_EXIT=0 + +assert_exit_code "validate failure: non-zero exit returned" "1" "${val_exit}" + +# --------------------------------------------------------------------------- +# Test: multiple new entries all appear in pipeline +# --------------------------------------------------------------------------- +echo "--- multiple new entries" + +write_inventory "${OLD}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" +write_inventory "${NEW}" \ + "backport-aws-1.19" "aws" "1.19.5" "abc123" "false" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" \ + "backport-gcp-1.5" "gcp" "1.5.0" "ghi789" "false" + +run_generate "${OLD}" "${NEW}" "true" "" "${OUT}" + +assert_file_contains "multiple: aws step present" "backport-aws-2.0" "${OUT}" +assert_file_contains "multiple: gcp step present" "backport-gcp-1.5" "${OUT}" + +# --------------------------------------------------------------------------- +# Test: invalid old inventory returns non-zero +# --------------------------------------------------------------------------- +echo "--- invalid inventory" + +echo "not: yaml: backports" > "${OLD}" +write_inventory "${NEW}" \ + "backport-aws-2.0" "aws" "2.0.0" "def456" "false" + +inv_exit=0 +: > "${OUT}" +generate_trigger_pipeline "${OLD}" "${NEW}" "true" "" "${OUT}" || inv_exit=$? + +assert_exit_code "invalid old inventory: non-zero exit" "1" "${inv_exit}" + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo "" +echo "Results: ${pass} passed, ${fail} failed" +if [[ "${fail}" -gt 0 ]]; then + exit 1 +fi diff --git a/.buildkite/scripts/trigger_backport.sh b/.buildkite/scripts/trigger_backport.sh new file mode 100755 index 00000000000..07c960a649d --- /dev/null +++ b/.buildkite/scripts/trigger_backport.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Triggers the integrations-backport pipeline when .backports.yml changes. +# In PR builds: validates new entries via a dry run against the base branch. +# On push to main: creates backport branches for new entries. + +source .buildkite/scripts/common.sh +source .buildkite/scripts/trigger_backport_lib.sh + +set -euo pipefail + +main() { + add_bin_path + with_yq + with_mage + + local dry_run old_inventory_ref diff_from diff_to pr_number="" label + local new_entry_msg new_entry_hint + + if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then + if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "main" ]]; then + echo "Pull request does not target main (base branch: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}), skipping" + exit 0 + fi + dry_run="true" + label="dry-run" + local from to + from="$(get_from_changeset)" + to="$(get_to_changeset)" + diff_from="$(git merge-base "${from}" "${to}")" + diff_to="${to}" + old_inventory_ref="origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" + new_entry_msg="skipping dry-runs for initial entries" + new_entry_hint="To validate new entries, add them in a follow-up PR after this one merges." + else + if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then + echo "Not on main branch (branch: ${BUILDKITE_BRANCH}), skipping" + exit 0 + fi + dry_run="false" + label="create" + diff_from="HEAD^" + diff_to="HEAD" + old_inventory_ref="HEAD^" + new_entry_msg="skipping create for initial entries" + new_entry_hint="To create branches for these entries, trigger the integrations-backport pipeline manually." + with_github_cli + pr_number="" + pr_number="$(retry 3 resolve_pr_number "${BUILDKITE_COMMIT}")" || { + echo "Warning: could not resolve PR number after retries, PR comments will be skipped" >&2 + } + fi + + backports_yml_changed_exit=0 + backports_yml_changed "${diff_from}" "${diff_to}" || backports_yml_changed_exit=$? + if [[ "${backports_yml_changed_exit}" -eq 2 ]]; then + exit 1 + fi + if [[ "${backports_yml_changed_exit}" -ne 0 ]]; then + echo ".backports.yml not changed, skipping backport ${label} trigger" + exit 0 + fi + + echo "--- .backports.yml changed — finding new entries" + + OLD_INVENTORY="" + PIPELINE_FILE="" + + cleanup() { + local exit_code=$? + [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" + [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" + exit "${exit_code}" + } + trap cleanup EXIT + + OLD_INVENTORY="$(mktemp)" + NEW_INVENTORY=".backports.yml" + + if ! load_old_backports_inventory "${old_inventory_ref}" "${OLD_INVENTORY}"; then + echo ".backports.yml is new — ${new_entry_msg}" + echo "${new_entry_hint}" + exit 0 + fi + + PIPELINE_FILE="$(mktemp --suffix=.yml)" + + generate_trigger_pipeline "${OLD_INVENTORY}" "${NEW_INVENTORY}" "${dry_run}" "${pr_number}" "${PIPELINE_FILE}" + + if [[ ! -s "${PIPELINE_FILE}" ]]; then + echo "No new non-archived entries found, skipping ${label} trigger" + exit 0 + fi + + echo "--- Uploading ${label} trigger(s)" + cat "${PIPELINE_FILE}" + buildkite-agent pipeline upload "${PIPELINE_FILE}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi diff --git a/.buildkite/scripts/trigger_backport_dryrun.sh b/.buildkite/scripts/trigger_backport_dryrun.sh deleted file mode 100755 index 5405fd99013..00000000000 --- a/.buildkite/scripts/trigger_backport_dryrun.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -# Triggered from the main pipeline when .backports.yml changes in a PR. -# For each entry that is new (absent from the base branch), uploads a trigger -# step that runs the integrations-backport pipeline in DRY_RUN mode. - -source .buildkite/scripts/common.sh - -set -euo pipefail - -if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then - echo "Not a pull request, skipping backport dry-run trigger" - exit 0 -fi - -if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "main" ]]; then - echo "Pull request does not target main (base branch: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}), skipping backport dry-run trigger" - exit 0 -fi - -add_bin_path -with_yq -with_mage - -from="$(get_from_changeset)" -to="$(get_to_changeset)" -commit_merge="$(git merge-base "${from}" "${to}")" - -if ! git diff --name-only "${commit_merge}" "${to}" | grep -qE '^\.backports\.yml$'; then - echo ".backports.yml not changed, skipping backport dry-run trigger" - exit 0 -fi - -echo "--- .backports.yml changed — finding new entries" - -BASE_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" - -OLD_INVENTORY="" -PIPELINE_FILE="" - -cleanup() { - local exit_code=$? - [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" - [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" - exit "${exit_code}" -} -trap cleanup EXIT - -OLD_INVENTORY="$(mktemp)" -NEW_INVENTORY=".backports.yml" - -if ! git show "origin/${BASE_BRANCH}:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then - echo ".backports.yml is new on ${BASE_BRANCH} — skipping dry-runs for initial entries" - echo "To validate new entries, add them in a follow-up PR after this one merges." - exit 0 -fi - -if ! yq -e '.backports' "${OLD_INVENTORY}" > /dev/null; then - echo "ERROR: old inventory is not valid YAML or missing 'backports' key: ${OLD_INVENTORY}" - exit 1 -fi - -if ! yq -e '.backports' "${NEW_INVENTORY}" > /dev/null; then - echo "ERROR: new inventory is not valid YAML or missing 'backports' key: ${NEW_INVENTORY}" - exit 1 -fi - -PIPELINE_FILE="$(mktemp --suffix=.yml)" -entries_found=0 - -while IFS= read -r branch; do - entry=".backports[] | select(.branch == \"${branch}\")" - - active_exit=0 - mage CheckBackportBranchActive "${branch}" || active_exit=$? - if [[ "${active_exit}" -eq 2 ]]; then - echo "ERROR: failed to check active status for branch '${branch}'" - exit 1 - fi - if [[ "${active_exit}" -ne 0 ]]; then - echo " Skipping inactive entry: ${branch}" - continue - fi - - # Only trigger for entries that are new (absent from the old inventory). - # If the entry already existed, the git branch is already created; there is - # nothing to provision. This also covers re-activating a previously archived - # entry (archived:true → false): the branch exists, so no dry-run is needed. - old_branch="$(yq "${entry} | .branch" "${OLD_INVENTORY}")" - - if [[ -n "${old_branch}" ]]; then - echo " Skipping existing entry: ${branch} (already present in base branch)" - continue - fi - - pkg="$(yq "${entry} | .package" "${NEW_INVENTORY}")" - base_version="$(yq "${entry} | .base_version" "${NEW_INVENTORY}")" - base_commit="$(yq "${entry} | .base_commit" "${NEW_INVENTORY}")" - - echo " Queuing dry-run: ${branch} (package=${pkg} version=${base_version} base_commit=${base_commit})" - - if [[ "${entries_found}" -eq 0 ]]; then - printf 'steps:\n' > "${PIPELINE_FILE}" - fi - - cat >> "${PIPELINE_FILE}" <&2 + return 1 + fi + if [[ -n "${pr_number}" ]]; then + echo "Associated PR: #${pr_number}" >&2 + else + echo "No PR found for commit ${commit}" >&2 + fi + echo "${pr_number}" +} + +load_old_backports_inventory() { + local git_ref="$1" + local output_file="$2" + if ! git show "${git_ref}:.backports.yml" > "${output_file}" 2>/dev/null; then + echo "Note: .backports.yml not found at ref '${git_ref}'" >&2 + return 1 + fi +} + +backports_yml_changed() { + local from="$1" + local to="$2" + local changed + if ! changed="$(git diff --name-only "${from}" "${to}" -- .backports.yml)"; then + echo "ERROR: git diff failed for refs '${from}' '${to}'" >&2 + return 2 + fi + [[ -n "${changed}" ]] +} + +generate_trigger_pipeline() { + local old_inventory="$1" # path to the pre-change inventory + local new_inventory="$2" # path to the post-change inventory + local dry_run="$3" # "true" or "false" — passed to the triggered build + local pr_number="$4" # merged PR number, may be empty + local pipeline_file="$5" # output file; written only if new entries are found + + if ! yq -e '.backports' "${old_inventory}" > /dev/null 2>&1; then + echo "ERROR: old inventory is not valid YAML or missing 'backports' key: ${old_inventory}" >&2 + return 1 + fi + + if ! yq -e '.backports' "${new_inventory}" > /dev/null 2>&1; then + echo "ERROR: new inventory is not valid YAML or missing 'backports' key: ${new_inventory}" >&2 + return 1 + fi + + if ! mage ValidateBackportsInventory; then + echo "ERROR: new inventory failed schema validation" >&2 + return 1 + fi + + local label_prefix + if [[ "${dry_run}" == "true" ]]; then + label_prefix="Backport dry-run" + else + label_prefix="Backport create" + fi + + while IFS= read -r branch; do + local entry=".backports[] | select(.branch == \"${branch}\")" + + # Only trigger for entries that are new (absent from the old inventory). + # If the entry already existed, the branch is already created; nothing to provision. + # This also covers re-activating a previously archived entry (archived:true → false): + # the branch exists, so no trigger is needed. + local old_branch + old_branch="$(yq "${entry} | .branch" "${old_inventory}")" + if [[ -n "${old_branch}" ]]; then + echo " Skipping existing entry: ${branch} (already present)" + continue + fi + + local active_exit=0 + mage CheckBackportBranchActive "${branch}" || active_exit=$? + if [[ "${active_exit}" -eq 2 ]]; then + echo "ERROR: failed to check active status for branch '${branch}'" >&2 + return 1 + fi + if [[ "${active_exit}" -ne 0 ]]; then + echo " Skipping inactive entry: ${branch}" + continue + fi + + local pkg base_version base_commit + pkg="$(yq "${entry} | .package" "${new_inventory}")" + base_version="$(yq "${entry} | .base_version" "${new_inventory}")" + base_commit="$(yq "${entry} | .base_commit" "${new_inventory}")" + + echo " Queuing ${label_prefix}: ${branch} (package=${pkg} version=${base_version} base_commit=${base_commit})" + + if [[ ! -s "${pipeline_file}" ]]; then + printf 'steps:\n' > "${pipeline_file}" + fi + + cat >> "${pipeline_file}" <> "${pipeline_file}" <- +// backport--. // // where is one or more letters, digits, or underscores, and -// starts with a digit followed by digits, dots, and an optional -// trailing 'x' wildcard (e.g. "6.x" or "6.14.x"). +// is exactly . or .x (e.g. "3.17" or "6.x"). // Whitespace, quotes, colons, semicolons, and all other special characters // are not permitted. -var branchRE = regexp.MustCompile(`^backport-[a-zA-Z0-9_]+-[0-9][0-9.]*x?$`) +var branchRE = regexp.MustCompile(`^backport-[a-zA-Z0-9_]+-[0-9]+\.([0-9]+|x)$`) + +// legacyBranchNames lists branch names that predate the current naming convention +// and are intentionally exempt from the branchRE format check. +// Each entry must include a comment explaining why the exception exists. +var legacyBranchNames = map[string]struct{}{ + // backport-aws-7.15.0 used a three-component version before the + // backport--. convention was established. + "backport-aws-7.15.0": {}, + // backport-security_detection_engine-8.9.10 was a one-off exception to + // publish a missing package version; it used a patch-level version rather + // than the standard major.minor format. + "backport-security_detection_engine-8.9.10": {}, +} // ActiveResult is the result of a CheckActive call. type ActiveResult struct { @@ -111,6 +124,141 @@ func (e entry) activeResult(now time.Time) ActiveResult { return result } +// AddEntry inserts a new backport entry into the inventory at path. +// The branch name is derived as backport--.. +// archived is set to false and maintained_until to null. +// The entry is placed in sorted order: package name ascending, then version descending (newest first). +// packagesDir is the path to the packages/ directory used to verify that packageName names a real +// package. Pass an empty string to skip this check. +// Returns the derived branch name. +func AddEntry(path, packageName, baseVersion, baseCommit, packagesDir string) (string, error) { + v, err := semver.StrictNewVersion(baseVersion) + if err != nil { + return "", fmt.Errorf("invalid base_version %q: %w", baseVersion, err) + } + branch := fmt.Sprintf("backport-%s-%d.%d", packageName, v.Major(), v.Minor()) + + knownPackages, err := buildKnownPackages(packagesDir) + if err != nil { + return "", fmt.Errorf("loading packages from %s: %w", packagesDir, err) + } + if knownPackages != nil { + if _, ok := knownPackages[packageName]; !ok { + return "", fmt.Errorf("unknown package %q: not found under %s", packageName, packagesDir) + } + } + + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading inventory: %w", err) + } + + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return "", fmt.Errorf("parsing inventory: %w", err) + } + + seq, err := backportsSequenceNode(&doc) + if err != nil { + return "", err + } + + pos := entryInsertPos(seq, packageName, v) + newNode := newEntryNode(packageName, branch, baseVersion, baseCommit) + + updated := make([]*yaml.Node, len(seq.Content)+1) + copy(updated, seq.Content[:pos]) + updated[pos] = newNode + copy(updated[pos+1:], seq.Content[pos:]) + seq.Content = updated + + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode(&doc); err != nil { + return "", fmt.Errorf("marshaling inventory: %w", err) + } + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + return "", fmt.Errorf("writing inventory: %w", err) + } + return branch, nil +} + +// backportsSequenceNode navigates from the document root to the sequence node +// under the "backports" key. +func backportsSequenceNode(doc *yaml.Node) (*yaml.Node, error) { + if doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 { + return nil, fmt.Errorf("unexpected document structure") + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + return nil, fmt.Errorf("expected mapping at document root") + } + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Value == "backports" { + seq := root.Content[i+1] + if seq.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("'backports' is not a sequence") + } + return seq, nil + } + } + return nil, fmt.Errorf("'backports' key not found in inventory") +} + +// entryInsertPos returns the index at which a new entry with the given package +// and version should be inserted to keep the sequence sorted (package ascending, +// then version descending within the same package — newest first). +func entryInsertPos(seq *yaml.Node, newPkg string, newVer *semver.Version) int { + for i, node := range seq.Content { + pkg := mappingFieldValue(node, "package") + if pkg > newPkg { + return i + } + if pkg == newPkg { + ver, err := semver.StrictNewVersion(mappingFieldValue(node, "base_version")) + if err == nil && ver.LessThan(newVer) { + return i + } + } + } + return len(seq.Content) +} + +// mappingFieldValue returns the scalar value for the given key in a YAML mapping node. +func mappingFieldValue(node *yaml.Node, key string) string { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return node.Content[i+1].Value + } + } + return "" +} + +// newEntryNode builds a YAML mapping node for a new backport entry. +// base_version and base_commit are double-quoted to match the existing file style. +func newEntryNode(pkg, branch, baseVersion, baseCommit string) *yaml.Node { + scalar := func(value, tag string, style yaml.Style) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value, Style: style} + } + key := func(k string) *yaml.Node { return scalar(k, "!!str", 0) } + str := func(v string) *yaml.Node { return scalar(v, "!!str", 0) } + quoted := func(v string) *yaml.Node { return scalar(v, "!!str", yaml.DoubleQuotedStyle) } + + return &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Content: []*yaml.Node{ + key("package"), str(pkg), + key("branch"), str(branch), + key("base_version"), quoted(baseVersion), + key("base_commit"), quoted(baseCommit), + key("maintained_until"), scalar("null", "!!null", 0), + key("archived"), scalar("false", "!!bool", 0), + }, + } +} + // ValidateInventory reads the .backports.yml inventory at path and returns a // combined error listing every schema violation found across all entries. // @@ -162,9 +310,9 @@ func validateEntryFields(i int, e entry, knownPackages map[string]struct{}, pack if e.Branch == "" { errs = append(errs, fmt.Errorf("%s: missing required field 'branch'", id)) - } else if !branchRE.MatchString(e.Branch) { - errs = append(errs, fmt.Errorf("%s: invalid branch %q: must match backport-- "+ - "(letters/digits/underscores in package name, version starts with a digit; "+ + } else if _, isLegacy := legacyBranchNames[e.Branch]; !isLegacy && !branchRE.MatchString(e.Branch) { + errs = append(errs, fmt.Errorf("%s: invalid branch %q: must match backport--. "+ + "(letters/digits/underscores in package name; "+ "no whitespace, quotes, colons, semicolons or other special characters)", id, e.Branch)) } diff --git a/dev/backports/inventory_test.go b/dev/backports/inventory_test.go index 9ca5bfa11f8..59f9052fc62 100644 --- a/dev/backports/inventory_test.go +++ b/dev/backports/inventory_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) func writeTemp(t *testing.T, content string) string { @@ -220,7 +221,7 @@ func TestValidateInventory(t *testing.T) { `, }, { - title: "valid branch with x wildcard and minor", + title: "invalid branch — three-component version (major.minor.x)", contents: `backports: - package: aws branch: backport-aws-6.14.x @@ -228,6 +229,43 @@ func TestValidateInventory(t *testing.T) { base_commit: "5b593f6681" maintained_until: null archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid branch — three-component version (major.minor.patch)", + contents: `backports: + - package: aws + branch: backport-aws-1.2.3 + base_version: "1.2.3" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "legacy branch backport-aws-7.15.0 passes despite three-component version", + contents: `backports: + - package: aws + branch: backport-aws-7.15.0 + base_version: "7.15.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + }, + { + title: "legacy branch backport-security_detection_engine-8.9.10 passes despite three-component version", + contents: `backports: + - package: security_detection_engine + branch: backport-security_detection_engine-8.9.10 + base_version: "8.9.10" + base_commit: "5b593f6681" + maintained_until: null + archived: false `, }, { @@ -629,4 +667,160 @@ func TestCheckActiveFileNotFound(t *testing.T) { assert.Contains(t, err.Error(), "reading inventory") } +func readInventory(t *testing.T, path string) inventory { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + var inv inventory + require.NoError(t, yaml.Unmarshal(data, &inv)) + return inv +} + +func TestAddEntry(t *testing.T) { + const twoEntries = `backports: + - package: aws + branch: backport-aws-3.0 + base_version: "3.0.0" + base_commit: "def5678901" + maintained_until: null + archived: false + - package: aws + branch: backport-aws-1.19 + base_version: "1.19.5" + base_commit: "abc1234567" + maintained_until: null + archived: false +` + + t.Run("inserts between versions of the same package", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "aws", "2.1.0", "aabbccddee", "") + require.NoError(t, err) + assert.Equal(t, "backport-aws-2.1", branch) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + assert.Equal(t, "backport-aws-3.0", inv.Backports[0].Branch) + assert.Equal(t, "backport-aws-2.1", inv.Backports[1].Branch) + assert.Equal(t, "backport-aws-1.19", inv.Backports[2].Branch) + }) + + t.Run("inserts between packages alphabetically", func(t *testing.T) { + inv := `backports: + - package: aws + branch: backport-aws-1.0 + base_version: "1.0.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false + - package: gcp + branch: backport-gcp-1.0 + base_version: "1.0.0" + base_commit: "ffeeddccbb" + maintained_until: null + archived: false +` + path := writeTemp(t, inv) + branch, err := AddEntry(path, "elastic_agent", "2.3.0", "112233aabb", "") + require.NoError(t, err) + assert.Equal(t, "backport-elastic_agent-2.3", branch) + + result := readInventory(t, path) + require.Len(t, result.Backports, 3) + assert.Equal(t, "backport-aws-1.0", result.Backports[0].Branch) + assert.Equal(t, "backport-elastic_agent-2.3", result.Backports[1].Branch) + assert.Equal(t, "backport-gcp-1.0", result.Backports[2].Branch) + }) + + t.Run("appends when new entry is last", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "zz_pkg", "1.0.0", "aabbccddee", "") + require.NoError(t, err) + assert.Equal(t, "backport-zz_pkg-1.0", branch) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + assert.Equal(t, "backport-zz_pkg-1.0", inv.Backports[2].Branch) + }) + + t.Run("prepends when new entry is first", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "aaa_pkg", "1.0.0", "aabbccddee", "") + require.NoError(t, err) + assert.Equal(t, "backport-aaa_pkg-1.0", branch) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + assert.Equal(t, "backport-aaa_pkg-1.0", inv.Backports[0].Branch) + assert.Equal(t, "backport-aws-3.0", inv.Backports[1].Branch) + }) + + t.Run("derives branch from major.minor only", func(t *testing.T) { + path := writeTemp(t, twoEntries) + branch, err := AddEntry(path, "aws", "2.5.3", "aabbccddee", "") + require.NoError(t, err) + assert.Equal(t, "backport-aws-2.5", branch) + }) + + t.Run("new entry has correct fields", func(t *testing.T) { + path := writeTemp(t, twoEntries) + _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee", "") + require.NoError(t, err) + + inv := readInventory(t, path) + require.Len(t, inv.Backports, 3) + e := inv.Backports[1] + assert.Equal(t, "aws", e.Package) + assert.Equal(t, "backport-aws-2.1", e.Branch) + assert.Equal(t, "2.1.0", e.BaseVersion) + assert.Equal(t, "aabbccddee", e.BaseCommit) + assert.Nil(t, e.MaintainedUntil) + require.NotNil(t, e.Archived) + assert.False(t, *e.Archived) + }) + + t.Run("existing entries keep double-quoted style", func(t *testing.T) { + path := writeTemp(t, twoEntries) + _, err := AddEntry(path, "aws", "2.1.0", "aabbccddee", "") + require.NoError(t, err) + + out, _ := os.ReadFile(path) + content := string(out) + assert.Contains(t, content, `base_version: "1.19.5"`) + assert.Contains(t, content, `base_version: "3.0.0"`) + }) + + t.Run("header comment is preserved", func(t *testing.T) { + inv := `# Backport inventory header +# +backports: + - package: aws + branch: backport-aws-1.0 + base_version: "1.0.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false +` + path := writeTemp(t, inv) + _, err := AddEntry(path, "aws", "2.0.0", "ffeeddccbb", "") + require.NoError(t, err) + + out, _ := os.ReadFile(path) + assert.Contains(t, string(out), "# Backport inventory header") + }) + + t.Run("invalid base_version returns error", func(t *testing.T) { + path := writeTemp(t, twoEntries) + _, err := AddEntry(path, "aws", "not-a-version", "aabbccddee", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid base_version") + }) + + t.Run("file not found returns error", func(t *testing.T) { + _, err := AddEntry("/no/such/file.yml", "aws", "1.0.0", "aabbccddee", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "reading inventory") + }) +} + func ptr(s string) *string { return &s } diff --git a/magefile.go b/magefile.go index fa63677018e..19fc3957b2b 100644 --- a/magefile.go +++ b/magefile.go @@ -324,6 +324,25 @@ func IsVersionLessThanLogsDBGA(version string) error { return nil } +// AddBackportEntry adds a new entry to .backports.yml for the given package and +// base version. The branch name is derived as backport--., +// archived is set to false, and maintained_until to null. The base commit is +// resolved via dev/scripts/get_release_commit.sh. The entry is inserted in +// sorted order (by package name ascending, then by version descending — newest first). +func AddBackportEntry(packageName, baseVersion string) error { + baseCommit, err := sh.Output("bash", "dev/scripts/get_release_commit.sh", "-p", packageName, "-v", baseVersion) + if err != nil { + return fmt.Errorf("resolving base commit for %s@%s: %w", packageName, baseVersion, err) + } + commit := strings.TrimSpace(baseCommit) + branch, err := backports.AddEntry(".backports.yml", packageName, baseVersion, commit, "packages") + if err != nil { + return err + } + fmt.Printf("Added: branch=%s base_commit=%s\n", branch, commit) + return nil +} + // CheckBackportBranchActive reports whether a backport branch is active per .backports.yml. // Prints ": active" or ": inactive ()". // Pass -json for JSON output: mage CheckBackportBranchActive -json