diff --git a/.azure-pipelines/Dockerfile.sai-vpp-overlay b/.azure-pipelines/Dockerfile.sai-vpp-overlay new file mode 100644 index 00000000..ea9dd8a9 --- /dev/null +++ b/.azure-pipelines/Dockerfile.sai-vpp-overlay @@ -0,0 +1,21 @@ +ARG BASE_IMAGE=docker-sai-test-vpp:base +FROM ${BASE_IMAGE} + +COPY debs /tmp/vpp-debs + +RUN set -eux; \ + cp /usr/sbin/sysctl /usr/sbin/sysctl.real; \ + cp /usr/bin/true /usr/sbin/sysctl; \ + apt-get update; \ + apt-get install -y --reinstall --allow-downgrades --no-install-recommends \ + /tmp/vpp-debs/*.deb; \ + mv /usr/sbin/sysctl.real /usr/sbin/sysctl; \ + for deb in /tmp/vpp-debs/*.deb; do \ + package="$(dpkg-deb -f "$deb" Package)"; \ + expected_version="$(dpkg-deb -f "$deb" Version)"; \ + installed_version="$(dpkg-query -W -f='${Version}' "$package")"; \ + test "$installed_version" = "$expected_version"; \ + dpkg --verify "$package"; \ + done; \ + apt-get clean; \ + rm -rf /tmp/vpp-debs /var/lib/apt/lists/* \ No newline at end of file diff --git a/.azure-pipelines/build-sai-vpp-test-image-template.yml b/.azure-pipelines/build-sai-vpp-test-image-template.yml new file mode 100644 index 00000000..89298d7b --- /dev/null +++ b/.azure-pipelines/build-sai-vpp-test-image-template.yml @@ -0,0 +1,209 @@ +parameters: +- name: timeout + type: number + default: 60 + +- name: sairedis_run_id + type: string + default: '' + +- name: vpp_artifact_name + type: string + default: vpp-trixie + +- name: image_artifact_name + type: string + default: docker-sai-test-vpp-platform + +jobs: +- job: BuildSaiVppTestImage + displayName: Overlay PR VPP packages on SAI test image + timeoutInMinutes: ${{ parameters.timeout }} + + pool: + vmImage: ubuntu-22.04 + + steps: + - checkout: self + clean: true + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: ${{ parameters.vpp_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/vpp + displayName: Download current-run VPP packages + + - ${{ if eq(parameters.sairedis_run_id, '') }}: + - task: DownloadPipelineArtifact@2 + name: downloadSaiRedis + inputs: + source: specific + project: build + pipeline: Azure.sonic-sairedis + artifact: docker-sai-test-vpp + path: $(Build.ArtifactStagingDirectory)/sairedis + runVersion: latestFromBranch + runBranch: refs/heads/master + displayName: Download latest successful SAI VPP test image + + - ${{ if ne(parameters.sairedis_run_id, '') }}: + - task: DownloadPipelineArtifact@2 + name: downloadSaiRedis + inputs: + source: specific + project: build + pipeline: Azure.sonic-sairedis + artifact: docker-sai-test-vpp + path: $(Build.ArtifactStagingDirectory)/sairedis + runVersion: specific + runId: ${{ parameters.sairedis_run_id }} + displayName: Download specified SAI VPP test image + + - script: | + set -euxo pipefail + + vpp_dir="$(Build.ArtifactStagingDirectory)/vpp" + sairedis_dir="$(Build.ArtifactStagingDirectory)/sairedis" + context_dir="$(Build.ArtifactStagingDirectory)/overlay-context" + output_dir="$(Build.ArtifactStagingDirectory)/output" + deb_dir="$context_dir/debs" + base_image_tag="$(cat "$sairedis_dir/image-tag.txt")" + image_tag="docker-sai-test-vpp:$(Build.DefinitionName).$(Build.BuildNumber).platform-vpp" + + test -s "$sairedis_dir/docker-sai-test-vpp.gz" + test -s "$sairedis_dir/provenance.txt" + test -s "$sairedis_dir/SHA256SUMS" + for contract_file in \ + ci-matrix-tests.txt \ + ci-pass-tests.txt \ + evaluate_ci_baseline.py \ + gen_compatibility_matrix.py; do + test -s "$sairedis_dir/ci-contract/$contract_file" + done + ( + cd "$sairedis_dir" + sha256sum -c SHA256SUMS + ) + + rm -rf "$context_dir" "$output_dir" + mkdir -p "$deb_dir" "$output_dir/ci-contract" + + copy_one_deb() + { + local pattern="$1" + mapfile -t matches < <(find "$vpp_dir" -type f -name "$pattern" | sort) + if [[ "${#matches[@]}" -ne 1 ]]; then + echo "Expected exactly one $pattern package, found ${#matches[@]}" >&2 + printf ' %s\n' "${matches[@]}" >&2 + exit 1 + fi + cp -v "${matches[0]}" "$deb_dir/" + } + + copy_one_deb 'libvppinfra_*_amd64.deb' + copy_one_deb 'vpp_*_amd64.deb' + copy_one_deb 'vpp-plugin-core_*_amd64.deb' + copy_one_deb 'vpp-plugin-dpdk_*_amd64.deb' + cp -v .azure-pipelines/Dockerfile.sai-vpp-overlay "$context_dir/Dockerfile" + + docker load -i "$sairedis_dir/docker-sai-test-vpp.gz" + docker image inspect "$base_image_tag" >/dev/null + docker build --no-cache \ + --build-arg "BASE_IMAGE=$base_image_tag" \ + --label com.sonic.saivpp-ci=true \ + --label "com.sonic.platform-vpp.source=$(Build.SourceVersion)" \ + -t "$image_tag" \ + "$context_dir" + + base_packages="$output_dir/base-package-inventory.txt" + derived_packages="$output_dir/derived-package-inventory.txt" + docker run --rm --entrypoint dpkg-query "$base_image_tag" \ + -W -f='${binary:Package}\t${Version}\n' | sort > "$base_packages" + docker run --rm --entrypoint dpkg-query "$image_tag" \ + -W -f='${binary:Package}\t${Version}\n' | sort > "$derived_packages" + + package_changes="$output_dir/package-changes.txt" + comm -3 "$base_packages" "$derived_packages" > "$package_changes" + if awk ' + { + package = $1 + sub(/^[[:space:]]+/, "", package) + if (package != "libvppinfra" && + package != "vpp" && + package != "vpp-plugin-core" && + package != "vpp-plugin-dpdk") { + print + unexpected = 1 + } + } + END { exit unexpected } + ' "$package_changes"; then + : + else + echo "Derivative image changed packages outside the VPP overlay:" >&2 + cat "$package_changes" >&2 + exit 1 + fi + + package_identity="$output_dir/sairedis-package-identity.txt" + for package in \ + libsairedis \ + libsaivs \ + libsaimetadata \ + saiserver \ + python-saithrift; do + base_entry="$(grep -E "^${package}(:[^[:space:]]+)?[[:space:]]" "$base_packages")" + derived_entry="$(grep -E "^${package}(:[^[:space:]]+)?[[:space:]]" "$derived_packages")" + test -n "$base_entry" + test "$base_entry" = "$derived_entry" + echo "$derived_entry" >> "$package_identity" + done + + docker save "$image_tag" | gzip -c > "$output_dir/docker-sai-test-vpp.gz" + echo "$image_tag" > "$output_dir/image-tag.txt" + cp -a "$sairedis_dir/ci-contract/." "$output_dir/ci-contract/" + cp -v "$sairedis_dir/provenance.txt" "$output_dir/sairedis-provenance.txt" + cp -v "$sairedis_dir/SHA256SUMS" "$output_dir/SAIREDIS_SHA256SUMS" + + provenance="$output_dir/platform-vpp-provenance.txt" + { + echo "build_definition=$(Build.DefinitionName)" + echo "build_number=$(Build.BuildNumber)" + echo "build_id=$(Build.BuildId)" + echo "source_version=$(Build.SourceVersion)" + echo "source_branch=$(Build.SourceBranch)" + echo "sairedis_requested_run_id=${{ parameters.sairedis_run_id }}" + echo "sairedis_downloaded_build=$(downloadSaiRedis.BuildNumber)" + echo "sairedis_base_image=$base_image_tag" + echo "derived_image=$image_tag" + echo + echo "replacement_packages:" + for deb in "$deb_dir"/*.deb; do + echo "File=$(basename "$deb")" + dpkg-deb -f "$deb" Package Version Architecture + sha256sum "$deb" + done + } > "$provenance" + + cp -a "$deb_dir" "$output_dir/vpp-debs" + ( + cd "$output_dir" + sha256sum \ + docker-sai-test-vpp.gz \ + base-package-inventory.txt \ + derived-package-inventory.txt \ + image-tag.txt \ + package-changes.txt \ + platform-vpp-provenance.txt \ + sairedis-package-identity.txt \ + sairedis-provenance.txt \ + SAIREDIS_SHA256SUMS \ + ci-contract/* \ + vpp-debs/* > SHA256SUMS + ) + rm -rf "$context_dir" "$vpp_dir" "$sairedis_dir" + displayName: Build VPP PR SAI test image + + - publish: $(Build.ArtifactStagingDirectory)/output + artifact: ${{ parameters.image_artifact_name }} + displayName: Archive VPP PR SAI test image \ No newline at end of file diff --git a/.azure-pipelines/test-sai-vpp-template.yml b/.azure-pipelines/test-sai-vpp-template.yml new file mode 100644 index 00000000..2ccf5853 --- /dev/null +++ b/.azure-pipelines/test-sai-vpp-template.yml @@ -0,0 +1,134 @@ +parameters: +- name: timeout + type: number + default: 360 + +- name: image_artifact_name + type: string + default: docker-sai-test-vpp-platform + +- name: log_artifact_name + type: string + default: sai-vpp-test-results + +jobs: +- job: RunSaiVppMatrix + displayName: Run VPP SAI compatibility tests + timeoutInMinutes: ${{ parameters.timeout }} + + pool: sonictest + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: ${{ parameters.image_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/download + displayName: Download VPP PR SAI test image + + - script: | + set -u + mapfile -t stale_containers < <(sudo docker ps -aq --filter label=com.sonic.saivpp-ci=true) + if [[ "${#stale_containers[@]}" -gt 0 ]]; then + sudo docker rm -f "${stale_containers[@]}" + fi + mapfile -t stale_images < <(sudo docker image ls -q --filter label=com.sonic.saivpp-ci=true | sort -u) + if [[ "${#stale_images[@]}" -gt 0 ]]; then + sudo docker image rm -f "${stale_images[@]}" + fi + displayName: Clean stale VPP SAI test resources + + - script: | + set -euxo pipefail + + download_dir="$(Build.ArtifactStagingDirectory)/download" + results_dir="$(Build.ArtifactStagingDirectory)/sai-vpp-results" + xml_dir="$results_dir/xml" + log_dir="$results_dir/log" + image_tag="$(cat "$download_dir/image-tag.txt")" + + ( + cd "$download_dir" + sha256sum -c SHA256SUMS + ) + mkdir -p "$xml_dir" "$log_dir" + chmod 0777 "$xml_dir" "$log_dir" + sudo docker load -i "$download_dir/docker-sai-test-vpp.gz" + sudo docker image inspect "$image_tag" > "$results_dir/image-inspect.json" + + set +e + sudo docker run --rm --privileged \ + --name "saivpp-ci-$(Build.BuildId)-$(System.JobAttempt)" \ + --label com.sonic.saivpp-ci=true \ + -e PORT_COUNT=32 \ + -e ISOLATE_EACH_TEST=1 \ + -e STARTUP_TIMEOUT=180 \ + -v "$xml_dir:/test-results" \ + -v "$log_dir:/var/log" \ + "$image_tag" \ + sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test \ + 2>&1 | tee "$results_dir/run.log" + matrix_rc=${PIPESTATUS[0]} + set -e + + echo "$matrix_rc" > "$results_dir/matrix.rc" + sudo chown -R "$(id -u):$(id -g)" "$results_dir" + chmod -R a+rX "$results_dir" + echo "VPP SAI matrix exit code: $matrix_rc" + displayName: Run VPP SAI full matrix + + - script: | + set -euxo pipefail + + download_dir="$(Build.ArtifactStagingDirectory)/download" + results_dir="$(Build.ArtifactStagingDirectory)/sai-vpp-results" + contract_dir="$download_dir/ci-contract" + matrix_rc="$(cat "$results_dir/matrix.rc")" + + python3 -c 'import defusedxml' 2>/dev/null || sudo uv pip install --system defusedxml + python3 "$contract_dir/gen_compatibility_matrix.py" \ + "$results_dir/xml" "$results_dir/compatibility-matrix.md" + python3 "$contract_dir/evaluate_ci_baseline.py" \ + --xml-dir "$results_dir/xml" \ + --baseline "$contract_dir/ci-pass-tests.txt" \ + --expected "$contract_dir/ci-matrix-tests.txt" \ + --matrix-rc "$matrix_rc" \ + --report "$results_dir/baseline-report.txt" + displayName: Evaluate VPP SAI stable baseline + + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + searchFolder: $(Build.ArtifactStagingDirectory)/sai-vpp-results/xml + testResultsFiles: TEST-*.xml + testRunTitle: VPP SAI PTF - platform-vpp + failTaskOnFailedTests: false + failTaskOnMissingResultsFile: true + condition: always() + + - script: | + set -euxo pipefail + download_dir="$(Build.ArtifactStagingDirectory)/download" + results_dir="$(Build.ArtifactStagingDirectory)/sai-vpp-results" + cp -v "$download_dir/platform-vpp-provenance.txt" "$results_dir/" + cp -v "$download_dir/sairedis-provenance.txt" "$results_dir/" + cp -v "$download_dir/SHA256SUMS" "$results_dir/" + cp -v "$download_dir/SAIREDIS_SHA256SUMS" "$results_dir/" + displayName: Collect VPP SAI provenance + condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/sai-vpp-results + artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt) + displayName: Publish VPP SAI logs and results + condition: always() + + - script: | + set -u + tag_file="$(Build.ArtifactStagingDirectory)/download/image-tag.txt" + if [[ -f "$tag_file" ]]; then + image_tag="$(cat "$tag_file")" + sudo docker image rm "$image_tag" || true + fi + displayName: Clean VPP SAI test image + condition: always() \ No newline at end of file diff --git a/README.md b/README.md index 12e8de04..a91401e8 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,14 @@ Refer to the [Getting started](docs/README.getting-started.md) document in docs Refer to the [document](docs/README.sonic_vm.md) in docs directory for details. +# VPP SAI compatibility CI + +The Azure pipeline builds the Trixie VPP packages from the current commit and tests them with the VPP SAI unit-test framework published by `sonic-sairedis`. `BuildSaiVppTestImage` downloads an approved `docker-sai-test-vpp` artifact, installs only the current run's `libvppinfra`, `vpp`, `vpp-plugin-core`, and `vpp-plugin-dpdk` packages in a derivative image, verifies that every non-VPP Debian package is unchanged, and publishes combined provenance and checksums. + +`TestSaiVpp` runs the four-module compatibility matrix on the `sonictest` pool. The expected 85-selector matrix, 34-selector stable-pass baseline, evaluator, and matrix generator come from the same approved sairedis artifact as the base image, so this repository does not carry a copy of sairedis test policy. + +By default, the pipeline selects the latest successful `Azure.sonic-sairedis` master artifact. Set the optional `sairedis_run_id` pipeline parameter to an immutable Azure build ID to validate against a specific sairedis image. Privileged test execution must retain the repository's trusted-maintainer authorization policy for pull requests. + # Troubleshooting diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b6eecfdd..5aa5f1d2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,6 +32,9 @@ parameters: - name: arch type: string default: amd64 + - name: sairedis_run_id + type: string + default: '' variables: - name: BUILD_BRANCH @@ -52,3 +55,19 @@ stages: parameters: arch: amd64 debian_version: bookworm + +- stage: BuildSaiVppTestImage + displayName: Build VPP PR SAI test image + dependsOn: BuildVpp + condition: succeeded('BuildVpp') + jobs: + - template: .azure-pipelines/build-sai-vpp-test-image-template.yml + parameters: + sairedis_run_id: ${{ parameters.sairedis_run_id }} + +- stage: TestSaiVpp + displayName: Test VPP PR against SAI compatibility matrix + dependsOn: BuildSaiVppTestImage + condition: succeeded('BuildSaiVppTestImage') + jobs: + - template: .azure-pipelines/test-sai-vpp-template.yml