diff --git a/.azure-pipelines/build-docker-sai-test-vpp-template.yml b/.azure-pipelines/build-docker-sai-test-vpp-template.yml new file mode 100644 index 0000000000..10c3834e1f --- /dev/null +++ b/.azure-pipelines/build-docker-sai-test-vpp-template.yml @@ -0,0 +1,227 @@ +parameters: +- name: timeout + type: number + default: 60 + +- name: sairedis_artifact_name + type: string + +- name: swss_common_artifact_name + type: string + +- name: artifact_name + type: string + +- name: vpp_run_id + type: string + +jobs: +- job: BuildSaiTestVppImage + displayName: Build docker-sai-test-vpp + timeoutInMinutes: ${{ parameters.timeout }} + + pool: + vmImage: 'ubuntu-22.04' + + steps: + - checkout: self + clean: true + submodules: recursive + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: ${{ parameters.sairedis_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + **/libsairedis_*.deb + **/libsaivs_*.deb + **/libsaimetadata_*.deb + **/saiserverv2_*.deb + **/python-saithriftv2_*.deb + displayName: Download sonic sairedis deb packages + + - task: DownloadPipelineArtifact@2 + name: downloadSwssCommon + inputs: + source: specific + project: build + pipeline: Azure.sonic-swss-common + artifact: ${{ parameters.swss_common_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/download + runVersion: latestFromBranch + runBranch: refs/heads/$(BUILD_BRANCH) + allowPartiallySucceededBuilds: true + patterns: | + **/libswsscommon_*.deb + **/libswsscommon-dev_*.deb + displayName: Download sonic swss common deb packages + + - task: DownloadPipelineArtifact@2 + name: downloadCommonLib + inputs: + source: specific + project: build + pipeline: Azure.sonic-buildimage.common_libs + artifact: common-lib + path: $(Build.ArtifactStagingDirectory)/download + runVersion: latestFromBranch + runBranch: refs/heads/$(BUILD_BRANCH) + patterns: | + **/target/debs/trixie/libyang3_*.deb + **/target/debs/trixie/libpcre3_*.deb + displayName: Download sonic-buildimage common packages + + - task: DownloadPipelineArtifact@2 + name: downloadVpp + inputs: + source: specific + project: build + pipeline: sonic-net.sonic-platform-vpp + artifact: vpp-trixie + path: $(Build.ArtifactStagingDirectory)/download + runVersion: specific + runId: ${{ parameters.vpp_run_id }} + displayName: Download resolved sonic platform-vpp packages + + - script: | + set -euxo pipefail + + download_dir="$(Build.ArtifactStagingDirectory)/download" + deb_dir="$(System.DefaultWorkingDirectory)/.azure-pipelines/docker-sai-test-vpp/debs" + context_dir="$(Build.ArtifactStagingDirectory)/docker-context" + image_tag="docker-sai-test-vpp:$(Build.DefinitionName).$(Build.BuildNumber)" + ptf_dir="$(System.DefaultWorkingDirectory)/SAI/test/ptf" + sai_revision="$(git -C SAI rev-parse HEAD)" + ptf_revision="$(git -C "$ptf_dir" rev-parse HEAD)" + ptf_describe="$(git -C "$ptf_dir" describe --tags --long --always HEAD)" + ptf_version="$(python3 .azure-pipelines/docker-sai-test-vpp/derive_ptf_version.py "$ptf_dir")" + + if [[ "$(downloadVpp.BuildNumber)" != "${{ parameters.vpp_run_id }}" ]]; then + echo "Resolved VPP run ${{ parameters.vpp_run_id }}, downloaded $(downloadVpp.BuildNumber)" >&2 + exit 1 + fi + + rm -rf "$deb_dir" + mkdir -p "$deb_dir" + + copy_one_deb() + { + local pattern="$1" + mapfile -t matches < <(find "$download_dir" -type f -name "$pattern" | sort) + if [[ "${#matches[@]}" -ne 1 ]]; then + echo "Expected exactly one $pattern package, found ${#matches[@]}" >&2 + printf ' %s\n' "${matches[@]}" >&2 + exit 1 + fi + cp -v "${matches[0]}" "$deb_dir/" + } + + copy_optional_deb() + { + local pattern="$1" + mapfile -t matches < <(find "$download_dir" -type f -name "$pattern" | sort) + if [[ "${#matches[@]}" -gt 1 ]]; then + echo "Expected at most one $pattern package, found ${#matches[@]}" >&2 + printf ' %s\n' "${matches[@]}" >&2 + exit 1 + fi + if [[ "${#matches[@]}" -eq 1 ]]; then + cp -v "${matches[0]}" "$deb_dir/" + fi + } + + copy_one_deb 'libvppinfra_*_amd64.deb' + copy_one_deb 'vpp_*_amd64.deb' + copy_one_deb 'vpp-plugin-core_*_amd64.deb' + copy_one_deb 'vpp-plugin-dpdk_*_amd64.deb' + copy_one_deb 'libsaivs_*_amd64.deb' + copy_one_deb 'libsairedis_*_amd64.deb' + copy_one_deb 'libsaimetadata_*_amd64.deb' + copy_one_deb 'saiserverv2_*_amd64.deb' + copy_one_deb 'python-saithriftv2_*_amd64.deb' + copy_one_deb 'libswsscommon_*_amd64.deb' + copy_optional_deb 'libswsscommon-dev_*_amd64.deb' + copy_one_deb 'libyang3_*_amd64.deb' + copy_optional_deb 'libpcre3_*_amd64.deb' + + provenance="$(Build.ArtifactStagingDirectory)/provenance.txt" + { + echo "build_definition=$(Build.DefinitionName)" + echo "build_number=$(Build.BuildNumber)" + echo "build_id=$(Build.BuildId)" + echo "source_version=$(Build.SourceVersion)" + echo "source_branch=$(Build.SourceBranch)" + echo "sai_revision=$sai_revision" + echo "ptf_revision=$ptf_revision" + echo "ptf_describe=$ptf_describe" + echo "ptf_version=$ptf_version" + echo "vpp_resolved_run_id=${{ parameters.vpp_run_id }}" + echo "vpp_downloaded_build=$(downloadVpp.BuildNumber)" + echo "swss_common_downloaded_build=$(downloadSwssCommon.BuildNumber)" + echo "common_lib_downloaded_build=$(downloadCommonLib.BuildNumber)" + echo + echo "packages:" + for deb in "$deb_dir"/*.deb; do + echo "File=$(basename "$deb")" + dpkg-deb -f "$deb" Package Version Architecture + sha256sum "$deb" + done + } > "$provenance" + + rm -rf "$context_dir" + context_harness="$context_dir/.azure-pipelines/docker-sai-test-vpp" + mkdir -p "$context_harness" "$context_dir/SAI/test" + for harness_file in \ + Dockerfile \ + lanemap.ini \ + port-map.ini \ + ptf-port-map.ini \ + run_test.sh \ + sai.profile \ + swss_log_stdout_preload.cpp \ + vpp_startup.conf.template; do + cp -a ".azure-pipelines/docker-sai-test-vpp/$harness_file" "$context_harness/" + done + cp -a "$deb_dir" "$context_harness/" + cp -a SAI/test/ptf "$context_dir/SAI/test/" + cp -a SAI/test/sai_test "$context_dir/SAI/test/" + + build_args=( + --no-cache + --build-arg "PTF_VERSION=$ptf_version" + --label com.sonic.saivpp-ci=true + -f "$context_harness/Dockerfile" + -t "$image_tag" + ) + for proxy_var in http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY; do + if [[ -n "${!proxy_var:-}" ]]; then + build_args+=(--build-arg "$proxy_var=${!proxy_var}") + fi + done + docker build "${build_args[@]}" "$context_dir" + + echo "$image_tag" > "$(Build.ArtifactStagingDirectory)/image-tag.txt" + docker save "$image_tag" | gzip -c > "$(Build.ArtifactStagingDirectory)/docker-sai-test-vpp.gz" + contract_dir="$(Build.ArtifactStagingDirectory)/ci-contract" + mkdir -p "$contract_dir" + for contract_file in \ + ci-matrix-tests.txt \ + ci-pass-tests.txt \ + evaluate_ci_baseline.py \ + gen_compatibility_matrix.py; do + cp -v ".azure-pipelines/docker-sai-test-vpp/$contract_file" "$contract_dir/" + done + ( + cd "$(Build.ArtifactStagingDirectory)" + sha256sum \ + docker-sai-test-vpp.gz \ + image-tag.txt \ + provenance.txt \ + ci-contract/* > SHA256SUMS + ) + rm -rf "$download_dir" "$context_dir" + displayName: Build ${{ parameters.artifact_name }} + + - publish: $(Build.ArtifactStagingDirectory)/ + artifact: ${{ parameters.artifact_name }} + displayName: Archive VPP SAI test image diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 1c5e620082..11bc003f95 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -43,12 +43,27 @@ parameters: - name: debian_version type: string +- name: saithrift_v2 + type: boolean + default: false + +- name: depends_on + type: object + default: [] + +- name: vpp_run_id + type: string + default: '' + jobs: - job: + dependsOn: ${{ parameters.depends_on }} displayName: ${{ parameters.arch }} timeoutInMinutes: ${{ parameters.timeout }} variables: DIFF_COVER_CHECK_THRESHOLD: 80 + ${{ if ne(parameters.vpp_run_id, '') }}: + VPP_RUN_ID: $[ dependencies.ResolveVpp.outputs['resolveVppRun.VPP_RUN_ID'] ] ${{ if eq(parameters.run_unit_test, true) }}: DIFF_COVER_ENABLE: 'true' @@ -187,18 +202,32 @@ jobs: sudo dpkg -i $(find ./download -name *.deb) workingDirectory: $(Build.ArtifactStagingDirectory) displayName: "Install libyang from common lib" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: sonic-net.sonic-platform-vpp - artifact: vpp-${{ parameters.debian_version }} - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - allowPartiallySucceededBuilds: true - path: $(Build.ArtifactStagingDirectory)/download - displayName: "Download sonic platform-vpp deb packages" - condition: eq('${{ parameters.arch }}', 'amd64') + - ${{ if eq(parameters.vpp_run_id, '') }}: + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: sonic-net.sonic-platform-vpp + artifact: vpp-${{ parameters.debian_version }} + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + allowPartiallySucceededBuilds: true + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download sonic platform-vpp deb packages" + condition: eq('${{ parameters.arch }}', 'amd64') + - ${{ if ne(parameters.vpp_run_id, '') }}: + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: sonic-net.sonic-platform-vpp + artifact: vpp-${{ parameters.debian_version }} + runVersion: specific + runId: ${{ parameters.vpp_run_id }} + allowPartiallySucceededBuilds: true + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download specified sonic platform-vpp deb packages" + condition: eq('${{ parameters.arch }}', 'amd64') - script: | set -ex sudo dpkg -i download/libvppinfra-dev_*_${{ parameters.arch }}.deb @@ -273,6 +302,33 @@ jobs: gcovr -r ./ -e ".*/SAI/.*" -e ".+/json.hpp" -e "swss/.+" -e ".*/.libs/.*" -e ".*/debian/.*" -e "vslib/vpp/.*" --exclude-unreachable-branches --json-pretty -o coverage-all.json gcovr -a "coverage-*.json" -x --xml-pretty -o coverage.xml displayName: "Run sonic sairedis unit tests" + - ${{ if eq(parameters.saithrift_v2, true) }}: + - script: | + set -euxo pipefail + + sudo apt-get update + sudo apt-get install -qq -y \ + dh-exec \ + libthrift-dev \ + python3-setuptools \ + python3-thrift \ + thrift-compiler + + sudo apt-get install -y \ + ./libsaimetadata_1.0.0_${{ parameters.arch }}.deb \ + ./libsaimetadata-dev_1.0.0_${{ parameters.arch }}.deb \ + ./libsaivs_1.0.0_${{ parameters.arch }}.deb \ + ./libsaivs-dev_1.0.0_${{ parameters.arch }}.deb + + rm -f ./*saithriftv2*.deb ./saiserverv2*.deb + pushd SAI + SAITHRIFTV2=true SAITHRIFT_VER=v2 platform=vpp \ + dpkg-buildpackage -us -uc -b -j$(nproc) + popd + + test -s saiserverv2_0.9.4_${{ parameters.arch }}.deb + test -s python-saithriftv2_0.9.4_${{ parameters.arch }}.deb + displayName: "Build VPP SAI Thrift v2 packages" - publish: $(System.DefaultWorkingDirectory)/ artifact: ${{ parameters.artifact_name }} displayName: "Archive sonic sairedis debian packages" diff --git a/.azure-pipelines/docker-sai-test-vpp/.gitignore b/.azure-pipelines/docker-sai-test-vpp/.gitignore new file mode 100644 index 0000000000..2adbd7811a --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/.gitignore @@ -0,0 +1,7 @@ +# Local-only working notes and test artifacts (not published to remote). +# devdocs/ — progress/debug logs for agent/developer context across branches +# demodocs/ — stakeholder demo notes +# results/ — JUnit XML, run logs, compatibility matrices (update README pass list when changed) +devdocs/ +demodocs/ +results/ diff --git a/.azure-pipelines/docker-sai-test-vpp/Dockerfile b/.azure-pipelines/docker-sai-test-vpp/Dockerfile new file mode 100644 index 0000000000..5177660d31 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/Dockerfile @@ -0,0 +1,146 @@ +FROM debian:trixie + +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_BREAK_SYSTEM_PACKAGES=1 \ + PYTHONUNBUFFERED=1 \ + SAI_PROFILE=/etc/sai/sai.profile \ + SAISERVER_PORTMAP=/etc/sai/port-map.ini \ + PTF_PORTMAP=/etc/sai/ptf-port-map.ini \ + SAI_TEST_DIR=/sai_test \ + TEST_RESULTS_DIR=/test-results + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Build from the sonic-sairedis repository root: +# docker build -f .azure-pipelines/docker-sai-test-vpp/Dockerfile -t docker-sai-test-vpp . +COPY .azure-pipelines/docker-sai-test-vpp /opt/docker-sai-test-vpp +COPY SAI/test/ptf /opt/ptf +COPY SAI/test/sai_test /sai_test + +ARG PTF_VERSION + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ca-certificates \ + iproute2 \ + libboost-serialization1.83.0 \ + libpcap0.8 \ + libthrift-0.19.0t64 \ + libzmq5 \ + procps \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-thrift \ + python3-wheel \ + redis-server \ + redis-tools \ + tcpdump && \ + rm -rf /var/lib/apt/lists/* + +RUN test -n "${PTF_VERSION}" && \ + python3 -m pip install --no-cache-dir scapy==2.5.0 unittest-xml-reporting==3.2.0 && \ + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PTF="${PTF_VERSION}" \ + python3 -m pip install --no-cache-dir --ignore-installed /opt/ptf && \ + apt-get purge -y build-essential python3-dev && \ + apt-get autoremove -y + +RUN set -eux; \ + deb_dir=/opt/docker-sai-test-vpp/debs; \ + require_deb() { \ + local label="$1"; \ + shift; \ + local found=0; \ + local pattern; \ + for pattern in "$@"; do \ + if compgen -G "${deb_dir}/${pattern}" >/dev/null; then \ + found=1; \ + fi; \ + done; \ + if [[ "$found" -eq 0 ]]; then \ + echo "Missing ${label}; expected one of: $*" >&2; \ + exit 1; \ + fi; \ + }; \ + require_deb "VPP infra package" "libvppinfra_*.deb"; \ + require_deb "VPP package" "vpp_*.deb"; \ + require_deb "VPP core plugin package" "vpp-plugin-core_*.deb"; \ + require_deb "VPP DPDK plugin package" "vpp-plugin-dpdk_*.deb"; \ + require_deb "SAI virtual switch package" "libsaivs_*.deb"; \ + require_deb "SAI Redis package" "libsairedis_*.deb"; \ + require_deb "SAI metadata package" "libsaimetadata_*.deb"; \ + require_deb "SONiC SWSS common package" "libswsscommon_*.deb"; \ + require_deb "YANG runtime package" "libyang_*.deb" "libyang3_*.deb"; \ + require_deb "SAI thrift server package" "saiserver_*.deb" "saiserverv2_*.deb"; \ + require_deb "SAI Python thrift package" "python-saithrift_*.deb" "python-saithriftv2_*.deb"; \ + runtime_debs=(); \ + have_saiserverv2=0; \ + have_python_saithriftv2=0; \ + compgen -G "${deb_dir}/saiserverv2_*.deb" >/dev/null && have_saiserverv2=1; \ + compgen -G "${deb_dir}/python-saithriftv2_*.deb" >/dev/null && have_python_saithriftv2=1; \ + while IFS= read -r deb_file; do \ + deb_name="$(basename "$deb_file")"; \ + case "$deb_name" in \ + *-dbg_*.deb|*-dbgsym_*.deb|*-dev_*.deb) continue ;; \ + libyang3-tools_*.deb) continue ;; \ + saiserver_*.deb) [[ "$have_saiserverv2" -eq 1 ]] && continue ;; \ + python-saithrift_*.deb) [[ "$have_python_saithriftv2" -eq 1 ]] && continue ;; \ + esac; \ + runtime_debs+=("$deb_file"); \ + done < <(find "$deb_dir" -maxdepth 1 -type f -name '*.deb' | sort); \ + if [[ "${#runtime_debs[@]}" -eq 0 ]]; then \ + echo "No runtime .deb packages found under ${deb_dir}" >&2; \ + exit 1; \ + fi; \ + apt-get update; \ + cp /usr/sbin/sysctl /usr/sbin/sysctl.real; \ + cp /usr/bin/true /usr/sbin/sysctl; \ + apt-get install -y --no-install-recommends --allow-downgrades "${runtime_debs[@]}"; \ + find /usr/lib/python3/dist-packages -maxdepth 1 -type d -name 'saithrift-*.egg' -print > /usr/lib/python3/dist-packages/saithrift.pth; \ + test -s /usr/lib/python3/dist-packages/saithrift.pth; \ + python3 -c 'import sai_thrift'; \ + mv /usr/sbin/sysctl.real /usr/sbin/sysctl; \ + build_dev_debs=(); \ + for f in ${deb_dir}/libswsscommon-dev_*.deb; do \ + [[ -f "$f" ]] && build_dev_debs+=("$f"); \ + done; \ + if [[ ${#build_dev_debs[@]} -gt 0 ]]; then \ + apt-get install -y --no-install-recommends "${build_dev_debs[@]}"; \ + else \ + apt-get install -y --no-install-recommends libswsscommon-dev || true; \ + fi; \ + apt-get install -y --no-install-recommends g++ libhiredis-dev && \ + g++ -shared -fPIC -o /usr/local/lib/libswss_log_stdout.so \ + /opt/docker-sai-test-vpp/swss_log_stdout_preload.cpp -lswsscommon && \ + apt-get purge -y g++ libhiredis-dev && \ + if [[ ${#build_dev_debs[@]} -gt 0 ]]; then apt-get purge -y libswsscommon-dev; fi && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN install -d \ + /etc/sai \ + /run/vpp \ + /var/run/redis \ + /test-results \ + /usr/share/sonic/hwsku \ + /var/log && \ + install -m 0644 /opt/docker-sai-test-vpp/sai.profile /etc/sai/sai.profile && \ + install -m 0644 /opt/docker-sai-test-vpp/lanemap.ini /etc/sai/lanemap.ini && \ + install -m 0644 /opt/docker-sai-test-vpp/port-map.ini /etc/sai/port-map.ini && \ + install -m 0644 /opt/docker-sai-test-vpp/ptf-port-map.ini /etc/sai/ptf-port-map.ini && \ + install -m 0755 /opt/docker-sai-test-vpp/run_test.sh /usr/local/bin/run_test.sh && \ + ln -sf /usr/local/bin/run_test.sh /run_test.sh + +WORKDIR / + +# This is a single-purpose, --privileged test harness: the entrypoint starts VPP, +# Redis and saiserver, creates/destroys veth + PortChannel netdevs, and uses raw +# AF_PACKET sockets, all of which require root inside the container. It is run +# manually/in CI as a disposable test container, never as a deployed service, so a +# non-root USER would break the harness without adding meaningful isolation. +# nosemgrep: dockerfile.security.missing-user-entrypoint.missing-user-entrypoint +ENTRYPOINT ["/usr/local/bin/run_test.sh"] \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/README.md b/.azure-pipelines/docker-sai-test-vpp/README.md new file mode 100644 index 0000000000..725f87ea35 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/README.md @@ -0,0 +1,364 @@ +# docker-sai-test-vpp — VPP SAI Unit-Test Framework + +A self-contained, single-container framework for validating the **SAI API implementation of the VPP virtual-switch backend** (`libsaivs.so`) by running the OpenComputeProject (OCP) `sai_test` PTF suite against a real VPP dataplane. + +## a) Purpose and design + +### What it does + +The framework exercises SAI operations (create/set/get/remove of ports, RIFs, routes, neighbors, VLANs, FDB, LAGs, ECMP, …) through a Thrift RPC interface and verifies the resulting **data-plane** behavior by injecting and capturing packets on virtual interfaces. It is the test vehicle for finding gaps between the SAI contract and the VPP backend, and for producing a per-test **compatibility matrix**. + +### Architecture (one privileged container) + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ docker-sai-test-vpp (--privileged) │ +│ │ +│ ┌───────────┐ ┌──────────────┐ ┌────────────────────────┐ │ +│ │ VPP │ │ saiserver │ │ PTF test runner │ │ +│ │ af_packet │◄───►│ libsaivs.so │◄────►│ sai_test/*.py │ │ +│ │ linux_cp │ VPP │ (VPP SAI) │Thrift│ sai_thrift adapter │ │ +│ └─────┬─────┘ API └──────────────┘ :9092└───────────┬────────────┘ │ +│ │ AF_PACKET raw socket │ (AF_PACKET) │ +│ OEthernet0 ◄════ veth ════► OEth0_peer ──────────────┘ │ +│ … (32 pairs) │ +└──────────────────────────────────────────────────────────────────────┘ + EXISTING: VPP, libsaivs.so, sai_test, sai_thrift, PTF, af_packet/linux_cp + THIS HARNESS: Dockerfile, run_test.sh, sai.profile, *-map.ini +``` + +- **VPP** is the dataplane; `OEthernetX` are the VPP-facing kernel veth ends and `OEthX_peer` are the PTF-facing ends. `OEthernetX` represents an out-facing (wire) interface; the inside `EthernetX` (linux_cp TAP) faces the SONiC control plane. +- **saiserver** is a thin Thrift→SAI shim (port 9092) linked against `libsaivs.so`. +- **PTF** runs the OCP `sai_test` Python suite and does packet I/O on the `OEthX_peer` ends. +- **`run_test.sh`** is the container entrypoint and orchestrator: Redis → veth topology → VPP → saiserver → PTF. It writes the SONiC-to-VPP interface map to `/usr/share/sonic/hwsku/sonic_vpp_ifmap.ini`. PortChannel netdevs and LAG/SVI connected IPs are set up in sai_test `setUp()` via sai_test's `SIMULATE_SONIC` helper (`config/simulate_sonic.py`), not in `run_test.sh`. + +### Key design point — per-test isolation (default) and config-signature grouping + +The VPP SAI backend can build the switch + host-interfaces **once per saiserver process**. The OCP framework is built to configure a common T0 setup once and have subsequent tests reuse it (`common_configured=true`). But different test classes ask `T0TestBase.setUp` for *different* common configs (e.g. ECMP tests need next-hop groups), and rebuilding the full T0 config twice in one saiserver process can crash the backend. + +**Default (`ISOLATE_EACH_TEST=1`):** every requested test class runs in its **own** config group. Before each test, `run_test.sh` tears down and restarts Redis + VPP + saiserver, waits for the old VPP process to be fully reaped, then starts a fresh backend. Each test rebuilds its own common config (`common_configured=false`). This eliminates cross-test state contamination (ECMP `-6`/`-7` reuse artifacts, ordering-dependent flakes) at the cost of ~45–50 minutes for the full 4-module matrix (87 tests × ~30s recycle+rebuild each). + +**Grouped mode (`ISOLATE_EACH_TEST=0`):** `run_test.sh` parses each test class's `setUp` (resolving config kwargs through the class **inheritance chain**) to compute a common-config "signature", groups tests by signature, and for each group restarts the backend once: the first test in the group builds + persists that group's config, the rest reuse it (`common_configured=true`). This is ~9× faster but passes fewer tests when upstream workarounds are not present. + +Set `COMMON_CONFIGURED_REUSE=0` only for legacy single-invocation debugging (one ptf process, no grouping). + +### Supported tests + +The table below lists OCP `sai_test` classes that **pass** on the current VPP SAI backend (last validated **2026-07-21** against `sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test` on `sai_vpp_ut_phase3`, `ISOLATE_EACH_TEST=1`). It is the published substitute for a full compatibility matrix: only passing tests are listed. After a local matrix run, update this section when the pass set changes (see **Collecting results** below). + +| Module | Passing test classes (CI required) | +|---|---| +| `sai_ecmp_test` | `EcmpLagDisableTestV4`, `EcmpLagDisableTestV6`, `EcmpReuseLagRouteV4`, `EcmpReuseLagRouteV6`, `RemoveAllNextHopMemeberTestV4`, `RemoveNexthopGroupTestV4` | +| `sai_neighbor_test` | `AddHostRouteTest`, `AddHostRouteTestV6`, `NhopDiffPrefixRemoveLonger`, `NhopDiffPrefixRemoveLongerV6`, `NhopDiffPrefixRemoveShorter`, `NhopDiffPrefixRemoveShorterV6`, `NoHostRouteTestV6` | +| `sai_rif_test` | — | +| `sai_route_test` | `DefaultRouteV4Test`, `DefaultRouteV6Test`, `LagMultipleRouteTest`, `LagMultipleRoutev6Test`, `RemoveRouteV4Test`, `RouteDiffPrefixAddThenDeleteLongerV4Test`, `RouteDiffPrefixAddThenDeleteLongerV6Test`, `RouteDiffPrefixAddThenDeleteShorterV4Test`, `RouteDiffPrefixAddThenDeleteShorterV6Test`, `RouteRifTest`, `RouteRifv6Test`, `RouteSameSipDipv4Test`, `RouteSameSipDipv6Test`, `RouteUpdateTest`, `RouteUpdatev6Test`, `StaicSviMacFloodingTest`, `StaicSviMacFloodingV6Test` | + +**30** classes are required to pass the PR check (`ci-pass-tests.txt`). The harness plans **87** test targets across the four modules above; `gen_compatibility_matrix.py` may report a higher row count when a test produces both ERROR and FAIL JUnit entries. + +#### Flaky L3-over-LAG ECMP tests (run, not gated) + +Three additional `sai_ecmp_test` classes have passed in clean local matrix runs but are **dataplane-flaky** (hash/L3-over-LAG forwarding: pass/fail can flip run-to-run with no code change). They remain in the full matrix (`ci-matrix-tests.txt`) for visibility but are **not** in `ci-pass-tests.txt` and do not fail the PR check: + +| Module | Flaky test classes | +|---|---| +| `sai_ecmp_test` | `ReAddLagEcmpTestV4`, `RemoveLagEcmpTestV4`, `RemoveLagEcmpTestV6` | + +Promote one of these into `ci-pass-tests.txt` only after repeated clean runs show it is stable. + +### CI regression baseline + +`ci-matrix-tests.txt` is the expected set of 85 runnable selectors from the four-module plan (the 87 planned classes include two non-runnable base classes). `ci-pass-tests.txt` is the **30-selector** stable-pass subset used by the PR check. CI requires the observed JUnit selector set to match the matrix contract, then leaves failures outside the stable baseline visible while failing on a missing, failed, errored, or skipped baseline selector. + +`evaluate_ci_baseline.py` compares the JUnit directory with the baseline, reports newly passing selectors as promotion candidates, and treats missing or malformed results and harness exit codes of 2 or greater as infrastructure failures. Baseline changes are reviewed explicitly; CI never updates the file automatically. + +When a runnable test class is added, removed, or renamed in one of the four CI modules, update the sorted, fully qualified selectors in `ci-matrix-tests.txt` in the same reviewed change. + +## b) Building the framework + +The image bundles pre-built `.deb` packages from `debs/` (git-ignored locally). You only need to regenerate `.deb`s when the corresponding source changes. All examples below use the local image tag **`docker-sai-test-vpp:local`** (the `build_harness.sh` default). + +### Use cases + +The framework is designed to support three distinct deployment and testing scenarios: + +#### **Use case 1: Local dev in a `sonic-buildimage` workspace** +- **What changes:** `vslib/` C++ backend, `SAI/test/sai_test` Python tests, VPP packages, or harness scripts. +- **How dependencies are supplied:** Copy runtime `.deb`s from `target/debs//` into `docker-sai-test-vpp/debs/`, then build the image. The default suite today is **trixie** (see Dockerfile). +- **Status:** Supported today via `build_harness.sh` (below). Assumes the workspace layout `sonic-buildimage/src/sonic-sairedis`. VPP packages can come from `target/debs/trixie/` after a platform-vpp build, from a `sonic-platform-vpp` pipeline download, or from `VPP_DEB_DIR` when running the build script. + +#### **Use case 2: `sonic-sairedis` PR CI** +- **What changes:** `vslib/` C++ backend, harness files, or OCP tests. +- **How dependencies are supplied:** The pipeline's **Build** stage compiles and produces fresh `libsairedis` / `libsaivs` / `saiserver` / `python-saithrift` artifacts. Other runtime `.deb`s (`libswsscommon`, `libyang`, VPP) must be downloaded from existing pipeline artifacts — following the same pattern as `.azure-pipelines/build-docker-sonic-vs-template.yml` (swss-common pipeline, sonic-platform-vpp `vpp-trixie`, buildimage common libs). +- **VPP selection:** `BuildTrixie` resolves the latest successful VPP master run once, or uses the optional root `vpp_run_id` override. The resolved immutable run ID is used for both the compile-time VPP packages and the runtime packages installed by `BuildSaiTestVpp`. +- **Status:** Pipeline wiring is implemented by `BuildSaiTestVpp` and `TestSaiVpp`; Azure artifact authorization and burn-in are required before making the check mandatory. +- **Required runtime packages and typical artifact sources:** + +| Package glob | PR build produces? | Typical CI download source | +|---|---|---| +| `libsaivs_*`, `libsairedis_*`, `libsaimetadata_*`, `saiserverv2_*`, `python-saithriftv2_*` | Yes (this repo's Build stage) | Current pipeline job artifacts | +| `libswsscommon_*` | No | `Azure.sonic-swss-common` pipeline artifact | +| `libyang_*` | No | `sonic-buildimage` common-lib / VS build artifact | +| `libvppinfra_*`, `vpp_*`, `vpp-plugin-*` | No | `sonic-net.sonic-platform-vpp` artifact `vpp-trixie` | + +#### **Use case 3: `sonic-platform-vpp` PR CI** +- **What changes:** VPP `.deb`s only. +- **How dependencies are supplied:** Pull an approved `docker-sai-test-vpp` image and its CI contract from the `sonic-sairedis` pipeline artifact, then build a derivative image that reinstalls only the VPP runtime packages produced by the current `sonic-platform-vpp` run. +- **Status:** The producer artifact includes `ci-contract/` for the platform-vpp consumer; the consumer pipeline implementation and hosted validation are owned by `sonic-platform-vpp`. +- **Workflow:** The platform-vpp pipeline loads the approved image, overlays fresh `libvppinfra`, `vpp`, `vpp-plugin-core`, and `vpp-plugin-dpdk` packages, verifies that every non-VPP Debian package is unchanged, and runs the artifact's matrix and baseline evaluator. + +### Required `.deb` packages (validated by the Dockerfile) + +| Package glob | Source repo / how produced | +|---|---| +| `libvppinfra_*`, `vpp_*`, `vpp-plugin-core_*`, `vpp-plugin-dpdk_*` | VPP packages (from the `sonic-platform-vpp` build) | +| `libsaivs_*` | `sonic-sairedis` — the VPP SAI backend under test | +| `libsairedis_*`, `libsaimetadata_*` | `sonic-sairedis` | +| `libswsscommon_*` | `sonic-swss-common` | +| `libyang_*` / `libyang3_*` | `sonic-buildimage` (libyang3 on trixie) | +| `saiserver_*` / `saiserverv2_*` | `sonic-sairedis` SAI Thrift server | +| `python-saithrift_*` / `python-saithriftv2_*` | `sonic-sairedis` SAI Thrift Python client | + +All of these are staged in [`debs/`](debs/) (local-only). The Dockerfile installs every runtime `.deb` it finds there (skipping `-dbg`/`-dev`/`-dbgsym`). When both legacy and v2 saithrift packages are present, **`saiserverv2_*` and `python-saithriftv2_*` are preferred**. + +### Build script (use case 1) + +`build_harness.sh` automates staging from `sonic-buildimage` and building the image as **`docker-sai-test-vpp:local`**. It validates that all required runtime `.deb`s are present in `debs/` before `docker build`. If sonic-sairedis packages are missing, it runs the trixie `make` targets automatically (disable with `--no-auto-build`). VPP, `libswsscommon`, and `libyang`/`libyang3` are not auto-built — the script fails with hints if they are absent. + +```bash +cd /src/sonic-sairedis/.azure-pipelines/docker-sai-test-vpp +./build_harness.sh +``` + +Useful options: `--build-sairedis` (force-rebuild sairedis debs even when present), `--no-auto-build` (never invoke `make` for missing sairedis debs), `--no-stage-debs` (image rebuild only), `--vpp-deb-dir `, `--image-tag `. Run `./build_harness.sh --help` for the full list. + +### Regenerating the SAI `.deb`s manually (only when `vslib/` C++ changes) + +From the **buildimage repo root** (`sonic-buildimage`): + +```bash +cd + +# Force re-generation by removing the stale targets first +rm -f target/debs/trixie/libsairedis_*.deb target/debs/trixie/libsairedis-dev_*.deb \ + target/debs/trixie/libsaivs_*.deb target/debs/trixie/libsaivs-dev_*.deb + +# Build libsaivs / libsairedis +make target/debs/trixie/libsairedis_1.0.0_amd64.deb + +# Build saiserver + saithrift client (if those changed) +BLDENV=trixie make -f Makefile.work target/debs/trixie/libsaithrift-dev_0.9.4_amd64.deb + +# Stage the fresh packages into the harness build context (prefer v2 names on trixie) +cp target/debs/trixie/libsaivs_*.deb target/debs/trixie/libsaivs-dev_*.deb \ + target/debs/trixie/libsairedis_*.deb target/debs/trixie/libsairedis-dev_*.deb \ + target/debs/trixie/libsaimetadata_*.deb \ + target/debs/trixie/saiserverv2_*.deb target/debs/trixie/python-saithriftv2_*.deb \ + src/sonic-sairedis/.azure-pipelines/docker-sai-test-vpp/debs/ +``` + +Then run `./build_harness.sh` (or `./build_harness.sh --no-stage-debs` if `debs/` is already up to date). + +#### Trixie `libsaithrift-dev` build failure (workaround) + +On trixie, `make -f Makefile.work target/debs/trixie/libsaithrift-dev_0.9.4_amd64.deb` may fail (legacy `libthrift-0.11.0` / python2 debhelper dependency). `libsairedis` / `libsaivs` debs still build. If saithrift packaging fails: + +1. Build `libsairedis` as above (produces fresh `libsaivs` / `libsaimetadata`). +2. Build `saiserver` manually inside `sonic-slave-trixie` from the sairedis tree, or copy a known-good `saiserverv2_*.deb` into `debs/`. +3. Rebuild the harness image with `./build_harness.sh --no-stage-debs`. + +The image must contain a `saiserverv2` binary built against the same `libsaivs` as the staged debs — a stale saiserver deb will produce confusing Thrift/runtime failures. + +> Behind a corporate proxy, prefix `make` with your Docker build credentials, e.g. `DOCKER_CONFIG=`. VPP `.deb`s come from the `sonic-platform-vpp` pipeline; drop new ones into `debs/` or pass `--vpp-deb-dir`. `build_harness.sh` forwards proxy env vars to `docker build`. + +### Building the image manually + +Equivalent to `./build_harness.sh --no-stage-debs` after `debs/` is populated. From the **`sonic-sairedis`** directory: + +```bash +cd /src/sonic-sairedis + +docker build --no-cache \ + -f .azure-pipelines/docker-sai-test-vpp/Dockerfile \ + -t docker-sai-test-vpp:local . +``` + +If you are behind a proxy, pass it through (both lower- and upper-case): + +```bash +docker build --no-cache \ + --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy \ + --build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$https_proxy \ + --build-arg no_proxy=$no_proxy --build-arg NO_PROXY=$no_proxy \ + -f .azure-pipelines/docker-sai-test-vpp/Dockerfile \ + -t docker-sai-test-vpp:local . +``` + +> Rebuild scope: editing only `run_test.sh` / `sai_test/**` / harness templates needs an **image rebuild** only (fast). Editing `vslib/` C++ needs a **`.deb` rebuild** (above) first, then the image. + +### VPP startup plugins (`vpp_startup.conf.template`) + +The harness-generated VPP config enables plugins required by current VPP + libsaivs, including: + +- **`tap_plugin.so`** — needed for VPP 26.06 linux-cp host-interface creation. +- **`sflow_plugin.so`** — saiserver startup asserts if the sflow message-id base is missing when this plugin is disabled. + +Other enabled plugins (`af_packet`, `linux_cp`, `linux_nl`, `acl`, `vxlan`, …) match the VPP SAI dataplane bench. Do not trim these without validating saiserver startup and host-interface creation. + +## c) Running tests + +The container entrypoint is `run_test.sh`. Test selectors are PTF targets: `module` (e.g. `sai_route_test`) or `module.Class` (e.g. `sai_route_test.RouteRifTest`). `PORT_COUNT` sets the port/veth count (use 32 for the standard T0 topology). + +### Where the tests come from + +The OCP `sai_test` suite is baked into the image at `/sai_test` (copied from `SAI/test/sai_test/` in the repo). PTF and the SAI Thrift client come from `SAI/test/ptf` and the `python-saithrift` / `python-saithriftv2` package. `run_test.sh` discovers test classes under `/sai_test` automatically. + +### Run a single test + +```bash +docker run --rm --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:local sai_route_test.RouteRifTest +``` + +### Run several tests (one container, grouped or isolated per env) + +```bash +docker run --rm --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:local \ + sai_route_test.RouteRifTest sai_ecmp_test.EcmpHashFieldSportTestV4 +``` + +With default `ISOLATE_EACH_TEST=1`, each selector still gets its own fresh backend even when multiple classes are listed. + +### Run whole modules, or everything + +```bash +# whole modules (default isolation: ~45-50 min for all four) +docker run --rm --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:local sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test + +# faster grouped mode (~9x; fewer passes without upstream workarounds) +docker run --rm --privileged -e PORT_COUNT=32 -e ISOLATE_EACH_TEST=0 \ + docker-sai-test-vpp:local sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test + +# every discovered test class (no args) +docker run --rm --privileged -e PORT_COUNT=32 docker-sai-test-vpp:local +``` + +### Collecting results (JUnit XML) and updating the supported-test list + +PTF writes one JUnit-XML file per test into `/test-results`. By convention these land in the local-only `results/` tree (git-ignored; not published to the remote). Run from the `docker-sai-test-vpp/` directory and bind-mount `results/xml` out. + +For a full 4-module matrix, prefer **`nohup`** (or an equivalent detached logger) so a dropped SSH/IDE session does not kill the container mid-run. Tail the log file for progress — long runs produce megabytes of output and some IDE terminals stop updating while the run continues. + +```bash +cd /src/sonic-sairedis/.azure-pipelines/docker-sai-test-vpp +mkdir -p results/xml +rm -f results/xml/TEST-*.xml +nohup docker run --rm --privileged -e PORT_COUNT=32 \ + -v "$PWD/results/xml:/test-results" \ + docker-sai-test-vpp:local \ + sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test \ + > results/run.log 2>&1 & +tail -f results/run.log +``` + +### Monitor + +While a matrix run is in progress (from the `docker-sai-test-vpp/` directory): + +| | | +|---|---| +| **Log** | `results/run.log` (or whatever path you passed to `nohup` / `tee`) | +| **Progress** | `grep -oE '\[[0-9]+/87\]' results/run.log \| tail -1` | +| **Container** | `docker ps --filter ancestor=docker-sai-test-vpp:local` | + +The `87` in the progress grep matches the four-module isolation plan (`sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test`). For other selectors, use `grep -oE '\[[0-9]+/[0-9]+\]'` instead. + +Monitor progress without the full firehose: + +```bash +grep -oE '\[[0-9]+/[0-9]+\]' results/run.log | tail -1 +grep -E '^(OK|FAIL|ERROR) |\[run_test\] ===' results/run.log | tail -20 +``` + +Then build a local compatibility matrix with the bundled generator (defaults to the `results/` tree). The generator parses JUnit XML with `defusedxml` (hardened against XXE), so install it once if needed: + +```bash +pip install defusedxml +python3 gen_compatibility_matrix.py # writes results/compatibility-matrix.md +``` + +`gen_compatibility_matrix.py` walks `results/xml/TEST-*.xml` and writes a PASS/FAIL/ERROR/SKIP table with a count summary. You can also pass an explicit ` [output.md]` to point it elsewhere. + +Evaluate the same results against the PR baseline with the matrix process exit code (`0` for an all-pass matrix, `1` when test failures are present, or `2+` for setup failure): + +```bash +python3 evaluate_ci_baseline.py \ + --xml-dir results/xml \ + --baseline ci-pass-tests.txt \ + --expected ci-matrix-tests.txt \ + --matrix-rc 1 \ + --report results/baseline-report.txt +``` + +**Publishing pass results:** when repeated clean runs establish a newly stable pass, add its fully qualified selector to `ci-pass-tests.txt` and update the **Supported tests** section in this `README.md` in the same reviewed change. List only PASS classes and do not commit the generated matrix; working notes and deep-dive logs may be kept under `devdocs/` (also local-only and git-ignored). + +## d) Additional information + +Debug hold mode, environment variables, in-container log paths, and common pitfalls. + +### Debug mode (leave VPP / saiserver / veths alive for inspection) + +```bash +docker rm -f officesai-debug 2>/dev/null +docker run -d --name officesai-debug --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:local --debug sai_route_test.RouteRifTest + +# while the test runs, inspect VPP state: +docker exec officesai-debug vppctl show interface +docker exec officesai-debug vppctl show ip fib +docker exec officesai-debug vppctl show bond +docker cp officesai-debug:/var/log/saiserver.log ./saiserver.log +``` + +In `--debug` the container leaves the dataplane running after the test so you can use `vppctl`; remember to `docker rm -f officesai-debug` when done. + +### Environment knobs + +| Variable | Default | Meaning | +|---|---|---| +| `PORT_COUNT` | 32 | number of `OEthernetX`/`OEthX_peer` veth pairs | +| `ISOLATE_EACH_TEST` | 1 | 1 = fresh backend + own config per test; 0 = config-signature grouping with reuse | +| `COMMON_CONFIGURED_REUSE` | 1 | 1 = plan multi-target runs with grouping/isolation; 0 = legacy single ptf invocation | +| `SAI_PORT_UP_SHARED_WAIT` | 1 | 1 = poll all ports together in `port_configer.py` (seconds, not ~64s serial) | +| `SAI_PORT_UP_RETRIES` | 2 | shared-wait retry count (with `SAI_PORT_UP_SHARED_WAIT=1`) | +| `SAI_PORT_UP_POLL_INTERVAL` | 1 | seconds between shared-wait polls | +| `KEEP_VETHS_UP_SECONDS` | 120 | how long the per-group watchdog keeps VPP host-interfaces/veths up | +| `KEEP_VETHS_UP_INTERVAL` | 3 | seconds between watchdog `vppctl set interface state` batches | +| `LAG_COUNT` | 4 | PortChannel netdevs torn down at container exit | +| `MTU` | 9100 | veth MTU | +| `STARTUP_TIMEOUT` | 60 | seconds to wait for VPP / saiserver readiness | +| `THRIFT_PORT` | 9092 | saiserver Thrift listen port | +| `LAG_RIF_IPS` | 1 | enable LAG RIF connected-IP assignment in sai_test setUp (`SIMULATE_SONIC`) | +| `SVI_RIF_IPS` | 1 | enable SVI RIF connected-IP assignment in sai_test setUp | +| `SIMULATE_SONIC` | 1 | set by `run_test.sh`; enables sai_test's SONiC control-plane simulation (PortChannel netdevs + LAG/SVI RIF IPs) | +| `SIMULATE_SONIC_IPV6_CONTROL_SRC_MAC` | `00:77:66:55:44:00` | discard only simulated-router RS/MLDv2 startup frames from this source MAC; empty disables the filter | +| `TEST_FILTER` | — | alternative way to pass a single selector via env | + +These are read by `run_test.sh` at container start (defaults shown). + +### Logs inside the container + +- `/var/log/saiserver.log` — saiserver stdout/stderr **plus** `libsaivs` `SWSS_LOG_*` lines. After the SAI change that removed `swss::Logger` setup from `saiserver.cpp`, the harness routes `SWSS_LOG_*` to **stderr** via an `LD_PRELOAD` shim (`swss_log_stdout_preload.cpp` → `/usr/local/lib/libswss_log_stdout.so`; the `.so` name is historical). `run_test.sh` redirects saiserver `2>&1` into this file so backend traces still land here. Routing to stderr (not stdout) keeps `SWSS_LOG_ENTER` lines out of `swss::exec()` captured stdout (e.g. the `find_new_bond_id` shell pipeline used during LAG create). +- `/var/log/vpp.log`, `/var/log/vpp-startup.log` — VPP CLI history / stdout (crash backtraces). +- `/var/log/vpp-api-trace.txt` — decoded VPP binary-API trace (dumped at teardown). +- `/test-results/TEST-*.xml` — per-test JUnit results (bind-mount to the local `results/xml/` directory on the host). + +### Gotchas + +- The container must be `--privileged` (raw AF_PACKET sockets on veths/TAPs). +- The VPP SAI backend can build the switch once per saiserver process; `run_test.sh` restarts the backend per test (default) or per config-signature group — do not expect to re-run a full config build inside a single long-lived saiserver. +- **`stop_backend()` waits for child exit:** `terminate_process()` calls `wait` on VPP/saiserver/redis PIDs after they die so the next `start_vpp()` cannot overlap with a dying instance (avoids transient `Killed vpp` during per-test recycle). +- **`Set port...` / `Turn up ports...` looks hung:** each test's T0 common-config build spends ~10–20s on port admin-up and veth bring-up with little console output; this is normal, not a deadlock. +- **Long matrix runs:** use `nohup` + `tail -f results/run.log`; do not rely on IDE agent terminals for live progress on 87-test isolation runs. +- A benign `buffer: numa[1] falling back to non-hugepage backed buffer pool` line at teardown is a host hugepage-availability warning, not a test failure. diff --git a/.azure-pipelines/docker-sai-test-vpp/build_harness.sh b/.azure-pipelines/docker-sai-test-vpp/build_harness.sh new file mode 100755 index 0000000000..9c5a592255 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/build_harness.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +# Stage runtime .deb packages and build the docker-sai-test-vpp image. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SAIREDIS_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +BUILDIMAGE_ROOT="${SONIC_BUILDIMAGE:-$(cd "${SAIREDIS_ROOT}/../.." && pwd)}" +DEB_STAGING="${SCRIPT_DIR}/debs" +DOCKERFILE="${SAIREDIS_ROOT}/.azure-pipelines/docker-sai-test-vpp/Dockerfile" +BLDENV="${BLDENV:-trixie}" +IMAGE_TAG="${IMAGE_TAG:-docker-sai-test-vpp:local}" +STAGE_DEBS="${STAGE_DEBS:-1}" +BUILD_SAIREDIS_DEBS="${BUILD_SAIREDIS_DEBS:-0}" +AUTO_BUILD_SAIREDIS_DEBS="${AUTO_BUILD_SAIREDIS_DEBS:-1}" +VPP_DEB_DIR="${VPP_DEB_DIR:-}" +PTF_VERSION="${PTF_VERSION:-}" + +SAIREDIS_DEB_PATTERNS=( + 'libsaivs_*.deb' 'libsairedis_*.deb' 'libsaimetadata_*.deb' + 'saiserver_*.deb' 'saiserverv2_*.deb' + 'python-saithrift_*.deb' 'python-saithriftv2_*.deb' +) + +usage() +{ + cat <<'EOF' +Usage: build_harness.sh [options] + +Stage runtime .deb packages into docker-sai-test-vpp/debs/ (local-only, +git-ignored) and build the test image from the sonic-sairedis repo root. + +Options: + --no-stage-debs Skip copying .debs (use what is already in debs/) + --build-sairedis Force-rebuild libsairedis/libsaivs (+ saithrift) in + sonic-buildimage before staging + --no-auto-build Do not invoke make when sonic-sairedis .debs are + missing (validate only; fail with hints) + --bldenv Debian suite for target/debs (default: trixie) + --image-tag Docker image tag (default: docker-sai-test-vpp:local) + --vpp-deb-dir Directory of VPP .debs (default: search buildimage + target/debs/ and VPP_DEB_DIR env) + -h, --help Show this help + +Environment: + SONIC_BUILDIMAGE Path to sonic-buildimage root (auto-detected if unset) + VPP_DEB_DIR Extra directory to copy VPP .debs from + PTF_VERSION Override the version derived from SAI/test/ptf + AUTO_BUILD_SAIREDIS_DEBS 1 (default): build sairedis .debs when missing + http_proxy/https_proxy/no_proxy Passed through to docker build when set + +Examples: + # Use-case 1 (local dev): stage, auto-build sairedis if needed, build image + ./build_harness.sh + + # Force-rebuild vslib .debs, then stage + build + ./build_harness.sh --build-sairedis + + # Rebuild image only (debs/ already populated and complete) + ./build_harness.sh --no-stage-debs +EOF +} + +log() +{ + printf '[build_harness] %s\n' "$*" +} + +die() +{ + printf '[build_harness] ERROR: %s\n' "$*" >&2 + exit 1 +} + +deb_present() +{ + local dir="$1" + shift + local pattern + + [[ -d "$dir" ]] || return 1 + + for pattern in "$@"; do + shopt -s nullglob + local matches=("${dir}"/${pattern}) + shopt -u nullglob + if [[ ${#matches[@]} -gt 0 ]]; then + return 0 + fi + done + + return 1 +} + +copy_matching_debs() +{ + local src_dir="$1" + shift + local pattern + + [[ -d "$src_dir" ]] || return 0 + + for pattern in "$@"; do + shopt -s nullglob + local matches=("${src_dir}"/${pattern}) + shopt -u nullglob + if [[ "${#matches[@]}" -eq 0 ]]; then + continue + fi + cp -v "${matches[@]}" "${DEB_STAGING}/" + done +} + +stage_vpp_debs() +{ + local src_dir="$1" + copy_matching_debs "${src_dir}" \ + 'libvppinfra_*.deb' 'vpp_*.deb' 'vpp-plugin-core_*.deb' 'vpp-plugin-dpdk_*.deb' +} + +stage_sairedis_debs_from_buildimage() +{ + local deb_dir="${BUILDIMAGE_ROOT}/target/debs/${BLDENV}" + + [[ -d "${BUILDIMAGE_ROOT}" ]] || die "sonic-buildimage not found at ${BUILDIMAGE_ROOT}" + mkdir -p "${DEB_STAGING}" + + log "Staging sonic-sairedis .debs from ${deb_dir} into ${DEB_STAGING}" + copy_matching_debs "${deb_dir}" "${SAIREDIS_DEB_PATTERNS[@]}" +} + +stage_debs_from_buildimage() +{ + local deb_dir="${BUILDIMAGE_ROOT}/target/debs/${BLDENV}" + + [[ -d "${BUILDIMAGE_ROOT}" ]] || die "sonic-buildimage not found at ${BUILDIMAGE_ROOT}" + mkdir -p "${DEB_STAGING}" + + log "Staging runtime .debs from ${deb_dir} into ${DEB_STAGING}" + + stage_vpp_debs "${deb_dir}" + copy_matching_debs "${deb_dir}" \ + 'libsaivs_*.deb' 'libsairedis_*.deb' 'libsaimetadata_*.deb' \ + 'libswsscommon_*.deb' 'libswsscommon-dev_*.deb' \ + 'libyang_*.deb' 'libyang3_*.deb' 'libpcre3_*.deb' \ + 'saiserver_*.deb' 'saiserverv2_*.deb' \ + 'python-saithrift_*.deb' 'python-saithriftv2_*.deb' + + if [[ -n "${VPP_DEB_DIR}" ]]; then + log "Staging VPP .debs from VPP_DEB_DIR=${VPP_DEB_DIR}" + stage_vpp_debs "${VPP_DEB_DIR}" + fi +} + +sairedis_debs_satisfied() +{ + deb_present "${DEB_STAGING}" 'libsaivs_*.deb' \ + && deb_present "${DEB_STAGING}" 'libsairedis_*.deb' \ + && deb_present "${DEB_STAGING}" 'libsaimetadata_*.deb' \ + && deb_present "${DEB_STAGING}" 'saiserver_*.deb' 'saiserverv2_*.deb' \ + && deb_present "${DEB_STAGING}" 'python-saithrift_*.deb' 'python-saithriftv2_*.deb' +} + +build_sairedis_debs() +{ + local force="${1:-0}" + + [[ -d "${BUILDIMAGE_ROOT}" ]] || die "sonic-buildimage not found at ${BUILDIMAGE_ROOT}" + + if [[ "${force}" == "1" ]]; then + log "Force-rebuilding libsairedis/libsaivs in ${BUILDIMAGE_ROOT} (BLDENV=${BLDENV})" + ( + cd "${BUILDIMAGE_ROOT}" + rm -f "target/debs/${BLDENV}/libsairedis_"*.deb "target/debs/${BLDENV}/libsairedis-dev_"*.deb \ + "target/debs/${BLDENV}/libsaivs_"*.deb "target/debs/${BLDENV}/libsaivs-dev_"*.deb + ) + else + log "Building libsairedis/libsaivs in ${BUILDIMAGE_ROOT} (BLDENV=${BLDENV})" + fi + + ( + cd "${BUILDIMAGE_ROOT}" + if [[ "${BLDENV}" == bookworm ]]; then + NOTRIXIE=1 make "target/debs/${BLDENV}/libsairedis_1.0.0_amd64.deb" + NOTRIXIE=1 BLDENV="${BLDENV}" make -f Makefile.work \ + "target/debs/${BLDENV}/libsaithrift-dev_0.9.4_amd64.deb" + else + make "target/debs/${BLDENV}/libsairedis_1.0.0_amd64.deb" + BLDENV="${BLDENV}" make -f Makefile.work \ + "target/debs/${BLDENV}/libsaithrift-dev_0.9.4_amd64.deb" + fi + ) +} + +report_missing_debs() +{ + local -a hints=() + + if ! sairedis_debs_satisfied; then + hints+=( + "sonic-sairedis (libsaivs, libsairedis, libsaimetadata, saiserverv2, python-saithriftv2):" + " ./build_harness.sh --build-sairedis" + " (or omit --no-auto-build to build automatically when missing)" + ) + fi + + if ! deb_present "${DEB_STAGING}" 'libvppinfra_*.deb'; then + hints+=( + "VPP infra (libvppinfra_*.deb):" + " Build platform-vpp in sonic-buildimage, or download sonic-net.sonic-platform-vpp" + " pipeline artifact vpp-trixie, then re-run with --vpp-deb-dir or VPP_DEB_DIR" + ) + fi + if ! deb_present "${DEB_STAGING}" 'vpp_*.deb'; then + hints+=( + "VPP (vpp_*.deb): same sources as libvppinfra (platform-vpp build or vpp-trixie artifact)" + ) + fi + if ! deb_present "${DEB_STAGING}" 'vpp-plugin-core_*.deb'; then + hints+=( + "VPP core plugin (vpp-plugin-core_*.deb): same sources as libvppinfra" + ) + fi + if ! deb_present "${DEB_STAGING}" 'vpp-plugin-dpdk_*.deb'; then + hints+=( + "VPP DPDK plugin (vpp-plugin-dpdk_*.deb): same sources as libvppinfra" + ) + fi + if ! deb_present "${DEB_STAGING}" 'libswsscommon_*.deb'; then + hints+=( + "SONiC SWSS common (libswsscommon_*.deb):" + " From sonic-buildimage: make target/debs/${BLDENV}/libswsscommon_1.0.0_amd64.deb" + " Or download Azure.sonic-swss-common pipeline artifact, then re-run ./build_harness.sh" + ) + fi + if ! deb_present "${DEB_STAGING}" 'libyang_*.deb' 'libyang3_*.deb'; then + hints+=( + "YANG runtime (libyang_*.deb or libyang3_*.deb):" + " From sonic-buildimage: make target/debs/${BLDENV}/libyang3_3.12.2-1_amd64.deb" + " Or download sonic-buildimage.common_libs / VS build artifact, then re-run ./build_harness.sh" + ) + fi + + if [[ ${#hints[@]} -eq 0 ]]; then + return 0 + fi + + printf '[build_harness] ERROR: Missing required .debs in %s\n' "${DEB_STAGING}" >&2 + local line + for line in "${hints[@]}"; do + printf ' %s\n' "$line" >&2 + done + return 1 +} + +ensure_staged_debs() +{ + mkdir -p "${DEB_STAGING}" + + if ! sairedis_debs_satisfied && [[ "${AUTO_BUILD_SAIREDIS_DEBS}" == "1" ]] \ + && [[ "${BUILD_SAIREDIS_DEBS}" != "1" ]]; then + log "sonic-sairedis .debs missing; auto-building in sonic-buildimage" + build_sairedis_debs 0 + stage_sairedis_debs_from_buildimage + fi + + report_missing_debs || die "cannot build image until required .debs are in ${DEB_STAGING}" +} + +docker_build_args() +{ + local -a args=(docker build --no-cache -f "${DOCKERFILE}" -t "${IMAGE_TAG}" .) + local var + + args+=(--build-arg "PTF_VERSION=${PTF_VERSION}") + + for var in http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY; do + if [[ -n "${!var:-}" ]]; then + args+=(--build-arg "${var}=${!var}") + fi + done + + printf '%s\0' "${args[@]}" +} + +main() +{ + while [[ $# -gt 0 ]]; do + case "$1" in + --no-stage-debs) STAGE_DEBS=0 ;; + --build-sairedis) BUILD_SAIREDIS_DEBS=1 ;; + --no-auto-build) AUTO_BUILD_SAIREDIS_DEBS=0 ;; + --bldenv) shift; BLDENV="${1:?--bldenv requires a value}" ;; + --image-tag) shift; IMAGE_TAG="${1:?--image-tag requires a value}" ;; + --vpp-deb-dir) shift; VPP_DEB_DIR="${1:?--vpp-deb-dir requires a value}" ;; + -h|--help) usage; exit 0 ;; + *) die "unknown option: $1 (try --help)" ;; + esac + shift + done + + if [[ "${BUILD_SAIREDIS_DEBS}" == "1" ]]; then + build_sairedis_debs 1 + stage_sairedis_debs_from_buildimage + fi + + if [[ "${STAGE_DEBS}" == "1" ]]; then + stage_debs_from_buildimage + fi + + ensure_staged_debs + + if [[ -z "${PTF_VERSION}" ]]; then + PTF_VERSION="$(python3 "${SCRIPT_DIR}/derive_ptf_version.py" \ + "${SAIREDIS_ROOT}/SAI/test/ptf")" + fi + log "Using PTF version ${PTF_VERSION}" + + log "Building image ${IMAGE_TAG} from ${SAIREDIS_ROOT}" + ( + cd "${SAIREDIS_ROOT}" + readarray -d '' -t docker_args < <(docker_build_args) + "${docker_args[@]}" + ) + log "Done: ${IMAGE_TAG}" +} + +main "$@" diff --git a/.azure-pipelines/docker-sai-test-vpp/ci-matrix-tests.txt b/.azure-pipelines/docker-sai-test-vpp/ci-matrix-tests.txt new file mode 100644 index 0000000000..789f6d4d0f --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/ci-matrix-tests.txt @@ -0,0 +1,86 @@ +# Runnable selectors expected from the four-module SAIVPP CI matrix. +sai_ecmp_test.EcmpCoExistLagRouteV4 +sai_ecmp_test.EcmpCoExistLagRouteV6 +sai_ecmp_test.EcmpHashFieldDportTestV4 +sai_ecmp_test.EcmpHashFieldDportTestV6 +sai_ecmp_test.EcmpHashFieldProtoTestV4 +sai_ecmp_test.EcmpHashFieldProtoTestV6 +sai_ecmp_test.EcmpHashFieldSIPTestV4 +sai_ecmp_test.EcmpHashFieldSIPTestV6 +sai_ecmp_test.EcmpHashFieldSportTestV4 +sai_ecmp_test.EcmpHashFieldSportTestV6 +sai_ecmp_test.EcmpIngressDisableTestV4 +sai_ecmp_test.EcmpIngressDisableTestV6 +sai_ecmp_test.EcmpLagDisableTestV4 +sai_ecmp_test.EcmpLagDisableTestV6 +sai_ecmp_test.EcmpLagTwoLayersWithDiffHashOffsetTestV4 +sai_ecmp_test.EcmpLagTwoLayersWithDiffHashOffsetTestV6 +sai_ecmp_test.EcmpReuseLagRouteV4 +sai_ecmp_test.EcmpReuseLagRouteV6 +sai_ecmp_test.EcmpTwoLayersWithDiffHashOffsetTestV4 +sai_ecmp_test.EcmpTwoLayersWithDiffHashOffsetTestV6 +sai_ecmp_test.IngressNoDiffTestV4 +sai_ecmp_test.LagTwoLayersWithDiffHashOffsetTestV4 +sai_ecmp_test.LagTwoLayersWithDiffHashOffsetTestV6 +sai_ecmp_test.ReAddLagEcmpTestV4 +sai_ecmp_test.ReAddLagEcmpTestV6 +sai_ecmp_test.RemoveAllNextHopMemeberTestV4 +sai_ecmp_test.RemoveLagEcmpTestV4 +sai_ecmp_test.RemoveLagEcmpTestV6 +sai_ecmp_test.RemoveNexthopGroupTestV4 +sai_neighbor_test.AddHostRouteTest +sai_neighbor_test.AddHostRouteTestV6 +sai_neighbor_test.NhopDiffPrefixRemoveLonger +sai_neighbor_test.NhopDiffPrefixRemoveLongerV6 +sai_neighbor_test.NhopDiffPrefixRemoveShorter +sai_neighbor_test.NhopDiffPrefixRemoveShorterV6 +sai_neighbor_test.NoHostRouteTest +sai_neighbor_test.NoHostRouteTestV6 +sai_neighbor_test.RemoveAddNeighborTestIPV4 +sai_neighbor_test.RemoveAddNeighborTestIPV6 +sai_rif_test.IngressDisableTestV4 +sai_rif_test.IngressDisableTestV6 +sai_rif_test.IngressMacUpdateTest +sai_rif_test.IngressMacUpdateTestV6 +sai_rif_test.IngressMtuTestV4 +sai_rif_test.IngressMtuTestV6 +sai_route_test.DefaultRouteV4Test +sai_route_test.DefaultRouteV6Test +sai_route_test.DropRouteTest +sai_route_test.DropRoutev6Test +sai_route_test.LagMultipleRouteTest +sai_route_test.LagMultipleRoutev6Test +sai_route_test.RemoveRouteV4Test +sai_route_test.RouteDiffPrefixAddThenDeleteLongerV4Test +sai_route_test.RouteDiffPrefixAddThenDeleteLongerV6Test +sai_route_test.RouteDiffPrefixAddThenDeleteShorterV4Test +sai_route_test.RouteDiffPrefixAddThenDeleteShorterV6Test +sai_route_test.RouteLPMRouteNexthopTest +sai_route_test.RouteLPMRouteNexthopv6Test +sai_route_test.RouteLPMRouteRifTest +sai_route_test.RouteLPMRouteRifv6Test +sai_route_test.RouteRifTest +sai_route_test.RouteRifv6Test +sai_route_test.RouteSameSipDipv4Test +sai_route_test.RouteSameSipDipv6Test +sai_route_test.RouteUpdateTest +sai_route_test.RouteUpdatev6Test +sai_route_test.StaicSviMacFloodingTest +sai_route_test.StaicSviMacFloodingV6Test +sai_route_test.SviDirectBroadcastTest +sai_route_test.SviMacAgeAfterMoveV4Test +sai_route_test.SviMacAgeAfterMoveV6Test +sai_route_test.SviMacAgingTest +sai_route_test.SviMacAgingV6Test +sai_route_test.SviMacFloodingTest +sai_route_test.SviMacFloodingv6Test +sai_route_test.SviMacLarningAfterageV4Test +sai_route_test.SviMacLarningAfterAgeV6Test +sai_route_test.SviMacLearningTest +sai_route_test.SviMacLearningV6Test +sai_route_test.SviMacMoveV4Test +sai_route_test.SviMacMoveV6Test +sai_route_test.SviMacrMoveStressV4Test +sai_route_test.SviMacrMoveStressV6Test +sai_route_test.SviRouteL3Test +sai_route_test.SviRouteL3v6Test diff --git a/.azure-pipelines/docker-sai-test-vpp/ci-pass-tests.txt b/.azure-pipelines/docker-sai-test-vpp/ci-pass-tests.txt new file mode 100644 index 0000000000..46f06c760f --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/ci-pass-tests.txt @@ -0,0 +1,31 @@ +# Stable SAIVPP PR baseline. Keep selectors fully qualified as module.Class. +sai_ecmp_test.EcmpLagDisableTestV4 +sai_ecmp_test.EcmpLagDisableTestV6 +sai_ecmp_test.EcmpReuseLagRouteV4 +sai_ecmp_test.EcmpReuseLagRouteV6 +sai_ecmp_test.RemoveAllNextHopMemeberTestV4 +sai_ecmp_test.RemoveNexthopGroupTestV4 +sai_neighbor_test.AddHostRouteTest +sai_neighbor_test.AddHostRouteTestV6 +sai_neighbor_test.NhopDiffPrefixRemoveLonger +sai_neighbor_test.NhopDiffPrefixRemoveLongerV6 +sai_neighbor_test.NhopDiffPrefixRemoveShorter +sai_neighbor_test.NhopDiffPrefixRemoveShorterV6 +sai_neighbor_test.NoHostRouteTestV6 +sai_route_test.DefaultRouteV4Test +sai_route_test.DefaultRouteV6Test +sai_route_test.LagMultipleRouteTest +sai_route_test.LagMultipleRoutev6Test +sai_route_test.RemoveRouteV4Test +sai_route_test.RouteDiffPrefixAddThenDeleteLongerV4Test +sai_route_test.RouteDiffPrefixAddThenDeleteLongerV6Test +sai_route_test.RouteDiffPrefixAddThenDeleteShorterV4Test +sai_route_test.RouteDiffPrefixAddThenDeleteShorterV6Test +sai_route_test.RouteRifTest +sai_route_test.RouteRifv6Test +sai_route_test.RouteSameSipDipv4Test +sai_route_test.RouteSameSipDipv6Test +sai_route_test.RouteUpdateTest +sai_route_test.RouteUpdatev6Test +sai_route_test.StaicSviMacFloodingTest +sai_route_test.StaicSviMacFloodingV6Test diff --git a/.azure-pipelines/docker-sai-test-vpp/derive_ptf_version.py b/.azure-pipelines/docker-sai-test-vpp/derive_ptf_version.py new file mode 100644 index 0000000000..e701efa450 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/derive_ptf_version.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Derive a deterministic PEP 440 version from an intact PTF checkout.""" + +import argparse +import pathlib +import re +import subprocess +import sys + + +DESCRIBE_RE = re.compile( + r"^v?(?P[0-9]+(?:\.[0-9]+)*)-" + r"(?P[0-9]+)-g(?P[0-9a-f]+)$" +) +VERSION_RE = re.compile(r"^[0-9]+(?:\.[0-9]+)*$") + + +def git(ptf_dir, *args): + result = subprocess.run( + ["git", "-C", str(ptf_dir), *args], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() + + +def version_from_describe(description): + match = DESCRIBE_RE.fullmatch(description) + if not match: + raise ValueError(f"unsupported git describe output: {description}") + + version = match.group("version") + distance = int(match.group("distance")) + if distance == 0: + return version + return f"{version}.post{distance}+g{match.group('revision')}" + + +def fallback_version(ptf_dir, revision): + version_file = pathlib.Path(ptf_dir) / "Version.txt" + if version_file.is_file(): + base_version = version_file.read_text(encoding="utf-8").strip() + if not VERSION_RE.fullmatch(base_version): + raise ValueError(f"invalid PTF Version.txt value: {base_version}") + return f"{base_version}+g{revision}" + + commit_count = git(ptf_dir, "rev-list", "--count", "HEAD") + return f"0.0.post{commit_count}+g{revision}" + + +def derive_version(ptf_dir): + if git(ptf_dir, "status", "--porcelain"): + raise ValueError(f"PTF checkout is dirty: {ptf_dir}") + + revision = git(ptf_dir, "rev-parse", "--short=7", "HEAD") + try: + description = git( + ptf_dir, + "describe", + "--tags", + "--long", + "--match", + "v[0-9]*", + "HEAD", + ) + except subprocess.CalledProcessError: + return fallback_version(ptf_dir, revision) + return version_from_describe(description) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("ptf_dir", type=pathlib.Path) + args = parser.parse_args(argv) + + try: + print(derive_version(args.ptf_dir)) + except (OSError, ValueError, subprocess.CalledProcessError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.azure-pipelines/docker-sai-test-vpp/evaluate_ci_baseline.py b/.azure-pipelines/docker-sai-test-vpp/evaluate_ci_baseline.py new file mode 100644 index 0000000000..e2981ad03b --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/evaluate_ci_baseline.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Evaluate the stable SAIVPP baseline against PTF JUnit XML results.""" + +import argparse +import glob +import os +import sys +from collections import defaultdict + +import defusedxml.ElementTree as ET + + +STATUS_ORDER = ("ERROR", "FAIL", "SKIP", "PASS") + + +def read_baseline(path): + selectors = [] + seen = set() + with open(path, encoding="utf-8") as baseline_file: + for line_number, raw_line in enumerate(baseline_file, 1): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + if line in seen: + raise ValueError(f"duplicate baseline selector at line {line_number}: {line}") + seen.add(line) + selectors.append(line) + if not selectors: + raise ValueError(f"baseline is empty: {path}") + return set(selectors) + + +def testcase_status(testcase): + if testcase.find("error") is not None: + return "ERROR" + if testcase.find("failure") is not None: + return "FAIL" + if testcase.find("skipped") is not None: + return "SKIP" + return "PASS" + + +def collect_results(xml_dir): + xml_paths = sorted(glob.glob(os.path.join(xml_dir, "TEST-*.xml"))) + if not xml_paths: + raise ValueError(f"no TEST-*.xml files found in {xml_dir}") + + statuses = defaultdict(list) + for xml_path in xml_paths: + try: + root = ET.parse(xml_path).getroot() + except Exception as exc: + raise ValueError(f"cannot parse {xml_path}: {exc}") from exc + + suites = [root] if root.tag == "testsuite" else root.iter("testsuite") + for suite in suites: + for testcase in suite.iter("testcase"): + selector = (testcase.get("classname") or "").strip() + if not selector: + module = (suite.get("name") or "").strip() + name = (testcase.get("name") or "").strip() + selector = f"{module}.{name}" if module and name else "" + if not selector: + raise ValueError(f"testcase without selector in {xml_path}") + statuses[selector].append(testcase_status(testcase)) + + if not statuses: + raise ValueError(f"no testcases found in {xml_dir}") + return statuses + + +def aggregate_status(statuses): + for status in STATUS_ORDER: + if status in statuses: + return status + raise ValueError(f"unknown test status list: {statuses}") + + +def format_report(baseline, results, matrix_rc): + observed = {selector: aggregate_status(statuses) + for selector, statuses in results.items()} + regressions = [] + for selector in sorted(baseline): + status = observed.get(selector) + if status is None: + regressions.append((selector, "MISSING")) + elif status != "PASS": + regressions.append((selector, status)) + + candidates = sorted(selector for selector, status in observed.items() + if status == "PASS" and selector not in baseline) + nonpasses = sorted((selector, status) for selector, status in observed.items() + if status != "PASS" and selector not in baseline) + + lines = [ + "SAIVPP CI baseline evaluation", + f"Matrix exit code: {matrix_rc}", + f"Baseline selectors: {len(baseline)}", + f"Observed selectors: {len(observed)}", + f"Stable baseline passes: {len(baseline) - len(regressions)}", + f"Regressions: {len(regressions)}", + f"New pass candidates: {len(candidates)}", + f"Known non-baseline non-passes: {len(nonpasses)}", + ] + if regressions: + lines.append("Regressions:") + lines.extend(f" {selector}: {status}" for selector, status in regressions) + if candidates: + lines.append("New pass candidates:") + lines.extend(f" {selector}" for selector in candidates) + if nonpasses: + lines.append("Known non-baseline non-passes:") + lines.extend(f" {selector}: {status}" for selector, status in nonpasses) + return "\n".join(lines) + "\n", regressions + + +def validate_matrix_contract(expected, results): + observed = set(results) + missing = sorted(expected - observed) + unexpected = sorted(observed - expected) + if not missing and not unexpected: + return None + + lines = ["Matrix selector contract mismatch"] + if missing: + lines.append("Missing expected selectors:") + lines.extend(f" {selector}" for selector in missing) + if unexpected: + lines.append("Unexpected selectors:") + lines.extend(f" {selector}" for selector in unexpected) + return "\n".join(lines) + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--xml-dir", required=True) + parser.add_argument("--baseline", required=True) + parser.add_argument("--expected", required=True) + parser.add_argument("--matrix-rc", type=int, default=0) + parser.add_argument("--report") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + try: + baseline = read_baseline(args.baseline) + expected = read_baseline(args.expected) + if not baseline <= expected: + unknown = ", ".join(sorted(baseline - expected)) + raise ValueError(f"baseline selectors are not in expected matrix: {unknown}") + if args.matrix_rc >= 2: + report = ( + "SAIVPP CI baseline evaluation\n" + f"Matrix exit code: {args.matrix_rc}\n" + "Infrastructure failure: matrix setup did not complete\n" + ) + regressions = [("", "INFRASTRUCTURE")] + else: + results = collect_results(args.xml_dir) + contract_error = validate_matrix_contract(expected, results) + if contract_error: + report = ( + "SAIVPP CI baseline evaluation\n" + f"Matrix exit code: {args.matrix_rc}\n" + f"Infrastructure failure: {contract_error}\n" + ) + regressions = [("", "INCOMPLETE")] + else: + report, regressions = format_report(baseline, results, args.matrix_rc) + except (OSError, ValueError) as exc: + report = f"SAIVPP CI baseline evaluation\nInfrastructure failure: {exc}\n" + regressions = [("", "INFRASTRUCTURE")] + + if args.report: + report_dir = os.path.dirname(os.path.abspath(args.report)) + os.makedirs(report_dir, exist_ok=True) + with open(args.report, "w", encoding="utf-8") as report_file: + report_file.write(report) + print(report, end="") + return 1 if regressions else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.azure-pipelines/docker-sai-test-vpp/gen_compatibility_matrix.py b/.azure-pipelines/docker-sai-test-vpp/gen_compatibility_matrix.py new file mode 100755 index 0000000000..af95cec542 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/gen_compatibility_matrix.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +""" +Generate a Markdown compatibility matrix from PTF JUnit-XML results. + +The VPP SAI unit-test harness (run_test.sh) writes one JUnit-XML file per test +into its --xunit-dir (mounted out as /test-results -> a host directory). By +convention that host directory is this harness's own results tree: + + docker-sai-test-vpp/results/xml/ <- per-test TEST-*.xml + docker-sai-test-vpp/results/compatibility-matrix.md <- generated matrix + +This script walks the TEST-*.xml files and emits a PASS/FAIL/ERROR/SKIP table. + +Usage: + # use the default results tree next to this script + python3 gen_compatibility_matrix.py + + # or point at an explicit xml dir / output file + python3 gen_compatibility_matrix.py [output.md] + +Example workflow: + cd docker-sai-test-vpp + mkdir -p results/xml + docker run --rm --privileged -e PORT_COUNT=32 \ + -v "$PWD/results/xml:/test-results" \ + docker-sai-test-vpp:local \ + sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test \ + 2>&1 | tee results/run.log + python3 gen_compatibility_matrix.py # writes results/compatibility-matrix.md +""" + +import sys +import os +import glob +import datetime + +# Use defusedxml (hardened against XXE / entity-expansion attacks) to parse the +# PTF JUnit XML. Install with `pip install defusedxml` if it is not already present. +import defusedxml.ElementTree as ET + +# Default results tree lives alongside this script, under docker-sai-test-vpp/. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_RESULTS_DIR = os.path.join(_SCRIPT_DIR, "results") +DEFAULT_XML_DIR = os.path.join(DEFAULT_RESULTS_DIR, "xml") +DEFAULT_OUTPUT = os.path.join(DEFAULT_RESULTS_DIR, "compatibility-matrix.md") + +ICON = { + "PASS": "\u2705 PASS", + "FAIL": "\u274c FAIL", + "ERROR": "\u26a0\ufe0f ERROR", + "SKIP": "\u23ed\ufe0f SKIP", +} + +# Result-status key (the icons used in the Result column). +RESULT_LEGEND = [ + ("\u2705 PASS", "test passed"), + ("\u274c FAIL", "assertion failed (e.g. expected packet not received)"), + ("\u26a0\ufe0f ERROR", "test errored out (exception / setUp / tearDown)"), + ("\u23ed\ufe0f SKIP", "test was skipped"), +] + +# SAI status codes that commonly appear in the Detail column. Values mirror +# SAI/inc/saistatus.h so a failing `status == -N` is readable at a glance. +SAI_STATUS_LEGEND = [ + ("0", "SAI_STATUS_SUCCESS"), + ("-1", "SAI_STATUS_FAILURE"), + ("-2", "SAI_STATUS_NOT_SUPPORTED"), + ("-3", "SAI_STATUS_NO_MEMORY"), + ("-4", "SAI_STATUS_INSUFFICIENT_RESOURCES"), + ("-5", "SAI_STATUS_INVALID_PARAMETER"), + ("-6", "SAI_STATUS_ITEM_ALREADY_EXISTS"), + ("-7", "SAI_STATUS_ITEM_NOT_FOUND"), + ("-8", "SAI_STATUS_BUFFER_OVERFLOW"), + ("-9", "SAI_STATUS_INVALID_PORT_NUMBER"), + ("-10", "SAI_STATUS_INVALID_PORT_MEMBER"), + ("-11", "SAI_STATUS_INVALID_VLAN_ID"), + ("-12", "SAI_STATUS_UNINITIALIZED"), + ("-13", "SAI_STATUS_TABLE_FULL"), + ("-14", "SAI_STATUS_MANDATORY_ATTRIBUTE_MISSING"), + ("-15", "SAI_STATUS_NOT_IMPLEMENTED"), + ("-16", "SAI_STATUS_ADDR_NOT_FOUND"), + ("-17", "SAI_STATUS_OBJECT_IN_USE"), +] + + +def collect_rows(xml_dir): + rows = [] + for path in sorted(glob.glob(os.path.join(xml_dir, "TEST-*.xml"))): + try: + root = ET.parse(path).getroot() + except Exception as e: # malformed XML -> surface, don't crash + rows.append(("?", os.path.basename(path), "PARSE-ERR", str(e)[:60])) + continue + suites = [root] if root.tag == "testsuite" else root.iter("testsuite") + for suite in suites: + for tc in suite.iter("testcase"): + mod = tc.get("classname", "") + name = tc.get("name", "") + fail = tc.find("failure") + err = tc.find("error") + skip = tc.find("skipped") + if err is not None: + status, detail = "ERROR", (err.get("message") or "") + elif fail is not None: + status, detail = "FAIL", (fail.get("message") or "") + elif skip is not None: + status, detail = "SKIP", (skip.get("message") or "") + else: + status, detail = "PASS", "" + detail = detail.replace("\n", " ").strip()[:80] + rows.append((mod, name, status, detail)) + rows.sort() + return rows + + +def render(rows): + counts = {} + for r in rows: + counts[r[2]] = counts.get(r[2], 0) + 1 + total = len(rows) + out = [] + out.append("# VPP SAI Compatibility Matrix\n") + generated = datetime.datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + out.append(f"_Generated: {generated}_\n") + + # Summary table: one row per status (in a stable, meaningful order) plus a + # total, with counts and percentages. + out.append("## Summary\n") + out.append("| Result | Count | % |") + out.append("|--------|------:|----:|") + order = ["PASS", "FAIL", "ERROR", "SKIP"] + seen = set(order) + for status in order + [s for s in sorted(counts) if s not in seen]: + n = counts.get(status, 0) + if n == 0 and status in seen: + continue + pct = (100.0 * n / total) if total else 0.0 + out.append(f"| {ICON.get(status, status)} | {n} | {pct:.1f}% |") + out.append(f"| **Total** | **{total}** | **100.0%** |") + out.append("") + + # Legend: result-status icons + SAI status codes seen in the Detail column. + out.append("## Legend\n") + out.append("**Result status**\n") + out.append("| Icon | Meaning |") + out.append("|------|---------|") + for icon, meaning in RESULT_LEGEND: + out.append(f"| {icon} | {meaning} |") + out.append("") + out.append("**SAI status codes** (see `SAI/inc/saistatus.h`) \u2014 appear in the Detail column\n") + out.append("| Code | Symbol |") + out.append("|------|--------|") + for code, symbol in SAI_STATUS_LEGEND: + out.append(f"| `{code}` | `{symbol}` |") + out.append("") + + out.append("## Results\n") + out.append("| Module | Test Class | Result | Detail |") + out.append("|--------|------------|--------|--------|") + for mod, name, status, detail in rows: + out.append(f"| `{mod}` | `{name}` | {ICON.get(status, status)} | {detail} |") + return "\n".join(out) + + +def main(argv): + args = argv[1:] + if args and args[0] in ("-h", "--help"): + sys.stderr.write(__doc__) + return 0 + + xml_dir = args[0] if len(args) >= 1 else DEFAULT_XML_DIR + # output: explicit 2nd arg, else default file when using the default xml dir, + # else stdout (so piping `> file.md` still works for a custom dir). + if len(args) >= 2: + output = args[1] + elif len(args) == 0: + output = DEFAULT_OUTPUT + else: + output = None # custom xml dir, no output given -> stdout + + if not os.path.isdir(xml_dir): + sys.stderr.write( + f"error: xml dir not found: {xml_dir}\n" + f" run the harness with -v /results/xml:/test-results first.\n" + ) + return 2 + + matrix = render(collect_rows(xml_dir)) + if output: + os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) + with open(output, "w", encoding="utf-8") as f: + f.write(matrix + "\n") + sys.stderr.write(f"wrote {output}\n") + else: + print(matrix) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.azure-pipelines/docker-sai-test-vpp/lanemap.ini b/.azure-pipelines/docker-sai-test-vpp/lanemap.ini new file mode 100644 index 0000000000..b9ec948707 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/lanemap.ini @@ -0,0 +1,34 @@ +# VPP wire-side interface to SAI lane mapping. +# Lanes match the default sai_test resources/config_db.json (Ethernet0,4,8,...,124). +OEthernet0:65,66,67,68 +OEthernet1:69,70,71,72 +OEthernet2:73,74,75,76 +OEthernet3:77,78,79,80 +OEthernet4:33,34,35,36 +OEthernet5:37,38,39,40 +OEthernet6:41,42,43,44 +OEthernet7:45,46,47,48 +OEthernet8:49,50,51,52 +OEthernet9:53,54,55,56 +OEthernet10:57,58,59,60 +OEthernet11:61,62,63,64 +OEthernet12:81,82,83,84 +OEthernet13:85,86,87,88 +OEthernet14:89,90,91,92 +OEthernet15:93,94,95,96 +OEthernet16:97,98,99,100 +OEthernet17:101,102,103,104 +OEthernet18:105,106,107,108 +OEthernet19:109,110,111,112 +OEthernet20:1,2,3,4 +OEthernet21:5,6,7,8 +OEthernet22:9,10,11,12 +OEthernet23:13,14,15,16 +OEthernet24:17,18,19,20 +OEthernet25:21,22,23,24 +OEthernet26:25,26,27,28 +OEthernet27:29,30,31,32 +OEthernet28:113,114,115,116 +OEthernet29:117,118,119,120 +OEthernet30:121,122,123,124 +OEthernet31:125,126,127,128 \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/port-map.ini b/.azure-pipelines/docker-sai-test-vpp/port-map.ini new file mode 100644 index 0000000000..726baadac3 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/port-map.ini @@ -0,0 +1,34 @@ +# saiserver front-panel alias to SAI lane mapping. +# Port names and lanes match the default sai_test resources/config_db.json. +Ethernet0 65,66,67,68 +Ethernet4 69,70,71,72 +Ethernet8 73,74,75,76 +Ethernet12 77,78,79,80 +Ethernet16 33,34,35,36 +Ethernet20 37,38,39,40 +Ethernet24 41,42,43,44 +Ethernet28 45,46,47,48 +Ethernet32 49,50,51,52 +Ethernet36 53,54,55,56 +Ethernet40 57,58,59,60 +Ethernet44 61,62,63,64 +Ethernet48 81,82,83,84 +Ethernet52 85,86,87,88 +Ethernet56 89,90,91,92 +Ethernet60 93,94,95,96 +Ethernet64 97,98,99,100 +Ethernet68 101,102,103,104 +Ethernet72 105,106,107,108 +Ethernet76 109,110,111,112 +Ethernet80 1,2,3,4 +Ethernet84 5,6,7,8 +Ethernet88 9,10,11,12 +Ethernet92 13,14,15,16 +Ethernet96 17,18,19,20 +Ethernet100 21,22,23,24 +Ethernet104 25,26,27,28 +Ethernet108 29,30,31,32 +Ethernet112 113,114,115,116 +Ethernet116 117,118,119,120 +Ethernet120 121,122,123,124 +Ethernet124 125,126,127,128 \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/ptf-port-map.ini b/.azure-pipelines/docker-sai-test-vpp/ptf-port-map.ini new file mode 100644 index 0000000000..d5143f7c99 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/ptf-port-map.ini @@ -0,0 +1,34 @@ +# PTF port index to SAI front-panel interface mapping. +# Port names match the default sai_test resources/config_db.json (Ethernet0,4,...,124). +0@Ethernet0 +1@Ethernet4 +2@Ethernet8 +3@Ethernet12 +4@Ethernet16 +5@Ethernet20 +6@Ethernet24 +7@Ethernet28 +8@Ethernet32 +9@Ethernet36 +10@Ethernet40 +11@Ethernet44 +12@Ethernet48 +13@Ethernet52 +14@Ethernet56 +15@Ethernet60 +16@Ethernet64 +17@Ethernet68 +18@Ethernet72 +19@Ethernet76 +20@Ethernet80 +21@Ethernet84 +22@Ethernet88 +23@Ethernet92 +24@Ethernet96 +25@Ethernet100 +26@Ethernet104 +27@Ethernet108 +28@Ethernet112 +29@Ethernet116 +30@Ethernet120 +31@Ethernet124 \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/run_test.sh b/.azure-pipelines/docker-sai-test-vpp/run_test.sh new file mode 100755 index 0000000000..fda3eae876 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/run_test.sh @@ -0,0 +1,1089 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Harness assets (e.g. vpp_startup.conf.template) live next to this script in the +# source tree and are installed under /opt/docker-sai-test-vpp in the image. When +# the image runs the script from /usr/local/bin, fall back to the install dir. +HARNESS_DIR="$SCRIPT_DIR" +[[ -f "$HARNESS_DIR/vpp_startup.conf.template" ]] || HARNESS_DIR="/opt/docker-sai-test-vpp" + +PORT_COUNT="${PORT_COUNT:-32}" +MTU="${MTU:-9100}" +# Number of PortChannel (LAG) netdevs used for cleanup at container exit. +LAG_COUNT="${LAG_COUNT:-4}" +# LAG/SVI connected-IP patterns consumed by sai_test's SIMULATE_SONIC helper +# (config/simulate_sonic.py), which assigns them in setUp when a LAG or VLAN RIF is +# created. These env vars retarget the helper's defaults to this VPP bench. +LAG_RIF_IPS="${LAG_RIF_IPS:-1}" +LAG_RIF_IPV4_PATTERN="${LAG_RIF_IPV4_PATTERN:-10.1.%d.1/24}" +LAG_RIF_IPV6_PATTERN="${LAG_RIF_IPV6_PATTERN:-fc00:1::%d:1/112}" +# VPP linux_cp host-interface (LCP tap) name prefix for a BondEthernet. +LAG_BE_TAP_PREFIX="${LAG_BE_TAP_PREFIX:-be}" + +# Assign the DUT-side connected IP for each SVI (VLAN) router interface (see +# simulate_sonic.assign_svi_rif_ips). Unlike a BondEthernet, a BVI has no linux_cp +# host-interface; the IP is set directly in VPP via vppctl once the BVI exists. +SVI_RIF_IPS="${SVI_RIF_IPS:-1}" +SVI_RIF_VLANS="${SVI_RIF_VLANS:-10:1 20:2}" +SVI_RIF_IPV4_PATTERN="${SVI_RIF_IPV4_PATTERN:-192.168.%d.1/24}" +SVI_RIF_IPV6_PATTERN="${SVI_RIF_IPV6_PATTERN:-fc02::%d:1/112}" +# VPP BVI interface name prefix for a VLAN SVI. +SVI_BVI_PREFIX="${SVI_BVI_PREFIX:-bvi}" +SAI_PROFILE="${SAI_PROFILE:-/etc/sai/sai.profile}" +SAISERVER_PORTMAP="${SAISERVER_PORTMAP:-/etc/sai/port-map.ini}" +PTF_PORTMAP="${PTF_PORTMAP:-/etc/sai/ptf-port-map.ini}" +SAI_TEST_DIR="${SAI_TEST_DIR:-/sai_test}" +TEST_RESULTS_DIR="${TEST_RESULTS_DIR:-/test-results}" +SONIC_VPP_IFMAP="${SONIC_VPP_IFMAP:-/usr/share/sonic/hwsku/sonic_vpp_ifmap.ini}" +THRIFT_PORT="${THRIFT_PORT:-9092}" +STARTUP_TIMEOUT="${STARTUP_TIMEOUT:-60}" +VPPCTL_TIMEOUT="${VPPCTL_TIMEOUT:-5}" +VPP_LOG="${VPP_LOG:-/var/log/vpp.log}" +VPP_STDOUT_LOG="${VPP_STDOUT_LOG:-/var/log/vpp-startup.log}" +VPP_API_TRACE="${VPP_API_TRACE:-/tmp/vpp-sai-api-trace.api}" +VPP_API_TRACE_TXT="${VPP_API_TRACE_TXT:-/var/log/vpp-api-trace.txt}" +SAISERVER_LOG="${SAISERVER_LOG:-/var/log/saiserver.log}" +REDIS_SOCKET="${REDIS_SOCKET:-/var/run/redis/redis.sock}" +REDIS_LOG="${REDIS_LOG:-/var/log/redis.log}" +LINKS_UP_MARKER="${LINKS_UP_MARKER:-/tmp/sai-vpp-links-up}" +LINK_UP_TRIGGER="${LINK_UP_TRIGGER:-Turn up ports...}" +KEEP_VETHS_UP_SECONDS="${KEEP_VETHS_UP_SECONDS:-120}" +KEEP_VETHS_UP_INTERVAL="${KEEP_VETHS_UP_INTERVAL:-3}" + +# Port admin-up wait tuning, consumed by the OCP test framework's +# turn_up_and_get_checked_ports() (SAI/test/sai_test/config/port_configer.py). +# In this VPP/veth harness the SAI port oper-status reads DOWN for the whole +# wait window (linux_cp carrier settles only after the config build), yet the +# links do come up and the dataplane works - so the framework's default +# per-port serial wait (32 ports x retries x interval ~= 64s) is pure dead time +# in every common-config build. We opt into the shared bounded wait (all ports +# polled together for at most retries x interval seconds) and a short interval, +# turning ~64s into a few seconds. These are exported so the ptf subprocess +# (and thus port_configer.py) sees them; unset = upstream/real-HW default. +export SAI_PORT_UP_SHARED_WAIT="${SAI_PORT_UP_SHARED_WAIT:-1}" +export SAI_PORT_UP_RETRIES="${SAI_PORT_UP_RETRIES:-2}" +export SAI_PORT_UP_POLL_INTERVAL="${SAI_PORT_UP_POLL_INTERVAL:-1}" + +# The SAI PTF T0 framework is designed to build the common switch configuration +# once and have subsequent tests reuse it (the persisted object IDs in +# /tmp/sai_model) by passing common_configured=true. When that path is NOT used, +# every test re-runs sai_create_switch + 32x sai_create_hostif inside the same +# long-lived saiserver/VPP process, which returns a null OID on the duplicate +# host-interface and crashes saiserver - so only the first test can run. With +# reuse enabled (default) the harness runs each test target as its own ptf +# invocation against one long-lived saiserver/VPP: the first builds + persists +# the common config (common_configured=false), the rest reuse it +# (common_configured=true). Set COMMON_CONFIGURED_REUSE=0 to fall back to a +# single ptf invocation for all targets (legacy behavior). +COMMON_CONFIGURED_REUSE="${COMMON_CONFIGURED_REUSE:-1}" + +# Per-test isolation. When set to 1 (default), every test target runs in its OWN +# config group: the backend (VPP + saiserver) is torn down and brought up fresh +# before each test, and each test rebuilds its own common config from scratch +# (common_configured=false) rather than reloading a `dut` persisted by a previous +# test. This eliminates all cross-test state contamination (the ECMP -6/-7 +# config-reuse artifacts and the v4/v6 NHG port-list aliasing), so the upstream OCP +# tests need no per-test workarounds. It is only affordable because the port-up +# wait was reduced from ~64s to ~6s (see SAI_PORT_UP_SHARED_WAIT and +# devdocs/progress-6-19.md); each test pays ~22s of recycle+rebuild. Set to 0 to +# fall back to config-signature grouping with persisted-config reuse (faster full +# runs, but tests then share a backend within a group). +ISOLATE_EACH_TEST="${ISOLATE_EACH_TEST:-1}" + +# Turn on sai_test's SONiC control-plane simulation (config/simulate_sonic.py): +# create PortChannel netdevs and assign LAG/SVI RIF IPs from the test setUp, since +# this standalone bench has no teamd / IntfMgr. The remaining vars retarget the +# helper's interface names / address patterns to this VPP bench. +export SIMULATE_SONIC=1 +export SIMULATE_SONIC_IPV6_CONTROL_SRC_MAC="${SIMULATE_SONIC_IPV6_CONTROL_SRC_MAC:-00:77:66:55:44:00}" +export LAG_RIF_IPS LAG_RIF_IPV4_PATTERN LAG_RIF_IPV6_PATTERN LAG_BE_TAP_PREFIX +export SVI_RIF_IPS SVI_RIF_VLANS SVI_RIF_IPV4_PATTERN SVI_RIF_IPV6_PATTERN SVI_BVI_PREFIX +# A VLAN SVI (BVI) has no host-interface netdev, so simulate_sonic cannot use "ip". +# Provide the VPP-specific command templates it runs to program the BVI address +# directly ({ifname}/{addr} are substituted). Keeping these here (the harness layer) +# is what lets sai_test's simulate_sonic stay backend-neutral. Note: the {ifname} +# braces can't sit inside a ${VAR:-default} expansion, so guard with a plain if. +if [[ -z "${SVI_RIF_PROBE_CMD:-}" ]]; then + SVI_RIF_PROBE_CMD="timeout ${VPPCTL_TIMEOUT} vppctl show interface {ifname}" +fi +if [[ -z "${SVI_RIF_SET_IP_CMD:-}" ]]; then + SVI_RIF_SET_IP_CMD="timeout ${VPPCTL_TIMEOUT} vppctl set interface ip address {ifname} {addr}" +fi +export SVI_RIF_PROBE_CMD SVI_RIF_SET_IP_CMD +export MTU VPPCTL_TIMEOUT + +DEBUG=0 +TEST_FILTER="${TEST_FILTER:-}" +TEST_FILTERS=() +VPP_PID="" +SAISERVER_PID="" +REDIS_PID="" +VPP_CONF="" +VPP_INIT_CLI="" +KEEP_VETHS_UP_PID="" + +log() +{ + echo "[run_test] $*" +} + +die() +{ + echo "[run_test] ERROR: $*" >&2 + exit 2 +} + +usage() +{ + cat <<'EOF' +Usage: run_test.sh [--debug] [TEST_FILTER ...] + +Examples: + run_test.sh + run_test.sh sai_route_test.RouteRifTest + run_test.sh sai_route_test.RouteRifTest sai_route_test.RouteRifv6Test + run_test.sh --debug sai_route_test.RouteRifTest + +Multiple TEST_FILTERs run sequentially. By default each target runs as its own +ptf invocation against one long-lived saiserver/VPP: the first builds and +persists the common config, the rest reuse it (common_configured=true). This is +required because the VPP SAI backend cannot re-create the switch/host-interfaces +twice in one process. Set COMMON_CONFIGURED_REUSE=0 to run all targets in a +single legacy invocation. + +Without any TEST_FILTER, every discovered test class under /sai_test runs (each +as its own reuse invocation). In debug mode, VPP, saiserver, and veth +interfaces are left running for vppctl inspection. +EOF +} + +parse_args() +{ + while [[ $# -gt 0 ]]; do + case "$1" in + --debug) + DEBUG=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + break + ;; + --*) + die "unknown option: $1" + ;; + *) + break + ;; + esac + done + + if [[ $# -gt 0 ]]; then + TEST_FILTERS=("$@") + fi +} + +exec_requested_shell() +{ + if [[ $# -eq 0 ]]; then + return 0 + fi + + case "$1" in + bash|sh|/bin/bash|/bin/sh) + exec "$@" + ;; + esac +} + +require_command() +{ + local command_name="$1" + + command -v "$command_name" >/dev/null 2>&1 || die "required command not found: $command_name" +} + +require_file() +{ + local file_path="$1" + + [[ -f "$file_path" ]] || die "required file not found: $file_path" +} + +vpp_interface_name() +{ + local port_index="$1" + + echo "OEthernet${port_index}" +} + +ptf_interface_name() +{ + local port_index="$1" + + echo "OEth${port_index}_peer" +} + +preflight() +{ + [[ "${EUID}" -eq 0 ]] || die "run_test.sh must run as root; use docker run --privileged" + + require_command ip + require_command vpp + require_command vppctl + require_command saiserver + require_command ptf + require_command redis-server + require_command redis-cli + require_command timeout + + require_file "$SAI_PROFILE" + require_file "$SAISERVER_PORTMAP" + require_file "$PTF_PORTMAP" + [[ -d "$SAI_TEST_DIR" ]] || die "required directory not found: $SAI_TEST_DIR" + + mkdir -p /run/vpp /var/log "$(dirname "$REDIS_SOCKET")" "$TEST_RESULTS_DIR" "$(dirname "$SONIC_VPP_IFMAP")" +} + +start_redis() +{ + log "Starting Redis on $REDIS_SOCKET" + rm -f "$REDIS_SOCKET" + + redis-server \ + --daemonize no \ + --bind 127.0.0.1 \ + --port 0 \ + --unixsocket "$REDIS_SOCKET" \ + --unixsocketperm 777 \ + --save '' \ + --appendonly no \ + --logfile "$REDIS_LOG" & + REDIS_PID="$!" + + for ((attempt = 1; attempt <= STARTUP_TIMEOUT; attempt++)); do + if [[ -n "$REDIS_PID" ]] && ! kill -0 "$REDIS_PID" >/dev/null 2>&1; then + die "Redis exited before becoming ready; see $REDIS_LOG" + fi + + if redis-cli -s "$REDIS_SOCKET" ping >/dev/null 2>&1; then + return 0 + fi + + sleep 1 + done + + die "timed out waiting for Redis; see $REDIS_LOG" +} + +delete_veths() +{ + set +e + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + local vpp_if + local ptf_if + + vpp_if="$(vpp_interface_name "$port_index")" + ptf_if="$(ptf_interface_name "$port_index")" + + if ip link show "$vpp_if" >/dev/null 2>&1; then + ip link delete "$vpp_if" >/dev/null 2>&1 + elif ip link show "$ptf_if" >/dev/null 2>&1; then + ip link delete "$ptf_if" >/dev/null 2>&1 + fi + done + set -e +} + +disable_ipv6_autoconf() +{ + # Disable IPv6 autoconfiguration on default and all interfaces. This prevents + # the Linux kernel from automatically generating IPv6 DAD and NDP neighbor/router + # solicitation packets when interfaces are brought up. Otherwise, at scale + # (PORT_COUNT=32), VPP raw sockets are flooded with unsolicited packets causing + # level-triggered poll event starvation of CLI and binary API connections. + echo 1 > /proc/sys/net/ipv6/conf/default/disable_ipv6 2>/dev/null || true + echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 2>/dev/null || true +} + +delete_portchannels() +{ + set +e + for ((lag_index = 0; lag_index < LAG_COUNT; lag_index++)); do + local pc_if="PortChannel${lag_index}" + if ip link show "$pc_if" >/dev/null 2>&1; then + ip link delete "$pc_if" >/dev/null 2>&1 + fi + done + set -e +} + +create_veths() +{ + log "Creating ${PORT_COUNT} OEthernet/OEth peer veth pair(s)" + delete_veths + rm -f "$LINKS_UP_MARKER" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + local vpp_if + local ptf_if + + vpp_if="$(vpp_interface_name "$port_index")" + ptf_if="$(ptf_interface_name "$port_index")" + + ip link add "$vpp_if" type veth peer name "$ptf_if" + ip link set dev "$vpp_if" mtu "$MTU" + ip link set dev "$ptf_if" mtu "$MTU" + done +} + +bring_up_veths() +{ + if [[ -f "$LINKS_UP_MARKER" ]]; then + return 0 + fi + + log "Bringing up ${PORT_COUNT} OEthernet/OEth peer veth pair(s)" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + local vpp_if + local ptf_if + + vpp_if="$(vpp_interface_name "$port_index")" + ptf_if="$(ptf_interface_name "$port_index")" + + ip link set dev "$vpp_if" up + ip link set dev "$ptf_if" up + done + + : > "$LINKS_UP_MARKER" +} + +# VPP host-interfaces (host-OEthernetX) are created administratively DOWN by the +# SAI hostif-creation path. While a host-interface is admin-DOWN, linux_cp keeps +# the paired kernel netdev (OEthernetX) in NO-CARRIER/M-DOWN, which drops the +# carrier on the PTF-side veth peer (OEthX_peer) and makes the dataplane unusable +# ("Network is down" on transmit/receive). +# +# Empirically, a single `set interface state host-OEthernetX up` in VPP is enough: +# VPP comes up, linux_cp propagates the state to the kernel netdev (takes a few +# seconds to settle) and brings the peer carrier up, and it stays up afterwards +# (no flapping). Setting the state again is idempotent. So run a short-lived +# background watchdog that re-asserts admin-up for all host-interfaces across the +# host-interface-creation window; interfaces that do not exist yet are simply +# reported as errors by vppctl and ignored. +keep_vpp_veths_up() +{ + local deadline=$((SECONDS + KEEP_VETHS_UP_SECONDS)) + local vpp_up_cmds + vpp_up_cmds="$(mktemp /tmp/vpp-sai-test-keepup.XXXXXX.cmds)" + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + echo "set interface state host-OEthernet${port_index} up" >> "$vpp_up_cmds" + done + + while [[ "$SECONDS" -lt "$deadline" ]]; do + timeout "$VPPCTL_TIMEOUT" vppctl exec "$vpp_up_cmds" >/dev/null 2>&1 || true + sleep "$KEEP_VETHS_UP_INTERVAL" + done + + rm -f "$vpp_up_cmds" +} + +create_sonic_vpp_ifmap() +{ + log "Writing SONiC-to-VPP interface map to $SONIC_VPP_IFMAP" + : > "$SONIC_VPP_IFMAP" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + echo "Ethernet$((port_index * 4)) host-OEthernet${port_index}" >> "$SONIC_VPP_IFMAP" + done +} + +generate_vpp_init_cli() +{ + VPP_INIT_CLI="$(mktemp /tmp/vpp-sai-test-init.XXXXXX.cli)" + : > "$VPP_INIT_CLI" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + echo "create host-interface name OEthernet${port_index}" >> "$VPP_INIT_CLI" + echo "set interface rx-mode host-OEthernet${port_index} interrupt" >> "$VPP_INIT_CLI" + done +} + +generate_vpp_config() +{ + VPP_CONF="$(mktemp /tmp/vpp-sai-test.XXXXXX.conf)" + local template="${HARNESS_DIR}/vpp_startup.conf.template" + local buffers_per_numa=$((PORT_COUNT * 2048)) + + sed -e "s|__VPP_LOG__|${VPP_LOG}|g" \ + -e "s|__BUFFERS_PER_NUMA__|${buffers_per_numa}|g" \ + -e "s|__VPP_INIT_CLI__|${VPP_INIT_CLI}|g" \ + "$template" > "$VPP_CONF" +} + +wait_for_vpp_ready() +{ + log "Waiting for VPP to become ready" + + for ((attempt = 1; attempt <= STARTUP_TIMEOUT; attempt++)); do + if [[ -n "$VPP_PID" ]] && ! kill -0 "$VPP_PID" >/dev/null 2>&1; then + die "VPP exited before becoming ready; see $VPP_LOG and $VPP_STDOUT_LOG" + fi + + if timeout "$VPPCTL_TIMEOUT" vppctl show version >/dev/null 2>&1; then + return 0 + fi + + sleep 1 + done + + die "timed out waiting for VPP; see $VPP_LOG and $VPP_STDOUT_LOG" +} + +verify_vpp_interfaces() +{ + log "Verifying VPP interfaces are created" + local interface_output + local port_index + local attempt + + for ((attempt = 1; attempt <= 15; attempt++)); do + local all_created=1 + interface_output="$(timeout "$VPPCTL_TIMEOUT" vppctl show interface 2>/dev/null || true)" + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + if ! grep -q "host-OEthernet${port_index}" <<< "$interface_output"; then + all_created=0 + break + fi + done + if [[ "$all_created" -eq 1 ]]; then + return 0 + fi + sleep 1 + done + + die "VPP interfaces were not created; see $VPP_LOG and $VPP_STDOUT_LOG" +} + +start_vpp() +{ + generate_vpp_init_cli + generate_vpp_config + log "Starting VPP" + vpp -c "$VPP_CONF" > "$VPP_STDOUT_LOG" 2>&1 & + VPP_PID="$!" + + wait_for_vpp_ready + verify_vpp_interfaces +} + +wait_for_saiserver_ready() +{ + log "Waiting for saiserver Thrift endpoint on 127.0.0.1:${THRIFT_PORT}" + + for ((attempt = 1; attempt <= STARTUP_TIMEOUT; attempt++)); do + if [[ -n "$SAISERVER_PID" ]] && ! kill -0 "$SAISERVER_PID" >/dev/null 2>&1; then + die "saiserver exited before becoming ready; see $SAISERVER_LOG" + fi + + if (echo >/dev/tcp/127.0.0.1/"$THRIFT_PORT") >/dev/null 2>&1; then + return 0 + fi + + sleep 1 + done + + die "timed out waiting for saiserver; see $SAISERVER_LOG" +} + +start_saiserver() +{ + log "Starting saiserver" + # libsaivs logs through SWSS_LOG_* (libswsscommon). saiserver.cpp no longer + # configures swss::Logger after the SAI standalone cleanup; LD_PRELOAD routes + # those lines to stdout so the redirect below lands in SAISERVER_LOG. + export LD_PRELOAD="/usr/local/lib/libswss_log_stdout.so${LD_PRELOAD:+:${LD_PRELOAD}}" + saiserver -p "$SAI_PROFILE" -f "$SAISERVER_PORTMAP" > "$SAISERVER_LOG" 2>&1 & + SAISERVER_PID="$!" + + wait_for_saiserver_ready +} + +ptf_target_requires_relax() +{ + case "$1" in + sai_fdb_test.BridgePortLearnDisableTest|\ + sai_fdb_test.BroadcastNoLearnTest|\ + sai_fdb_test.FdbAgingAfterMoveTest|\ + sai_fdb_test.FdbAgingTest|\ + sai_fdb_test.FdbFlushAllDynamicTest|\ + sai_fdb_test.FdbFlushAllStaticTest|\ + sai_fdb_test.FdbFlushAllTest|\ + sai_fdb_test.FdbFlushPortDynamicTest|\ + sai_fdb_test.FdbFlushPortStaticTest|\ + sai_fdb_test.FdbFlushVlanDynamicTest|\ + sai_fdb_test.FdbFlushVlanStaticTest|\ + sai_fdb_test.MulticastNoLearnTest|\ + sai_fdb_test.NonBridgePortNoLearnTest|\ + sai_fdb_test.RemoveVlanmemberLearnTest|\ + sai_fdb_test.VlanLearnDisableTest|\ + sai_route_test.SviDirectBroadcastTest|\ + sai_sanity_test.SaiSanityTest|\ + sai_tunnel_test.SviIPInIPTunnelDecapFloodV6InV4Test|\ + sai_tunnel_test.SviIPInIPTunnelDecapFloodv4Inv4Test|\ + sai_vlan_test.ArpRequestFloodingTest|\ + sai_vlan_test.BroadcastTest|\ + sai_vlan_test.DisableMacLearningTaggedTest|\ + sai_vlan_test.DisableMacLearningUntaggedTest|\ + sai_vlan_test.TaggedVlanFloodingTest|\ + sai_vlan_test.UnTaggedVlanFloodingTest) + return 0 + ;; + esac + + return 1 +} + +build_ptf_args() +{ + # Args: + # test_target a ptf test selector (module or module.Class), or empty + # for the whole /sai_test suite. + # common_configured "true" -> reuse the previously persisted common config + # "false" -> build (and persist) the common config + # "" -> do not pass the param at all (legacy) + # xunit_dir directory PTF writes its JUnit XML into. PTF rmtree's + # this dir on startup, so it must be a private per-call + # dir (NOT a bind mount, NOT shared across invocations). + local test_target="$1" + local common_configured="$2" + local xunit_dir="$3" + local test_params="thrift_server='127.0.0.1';port_map_file='$PTF_PORTMAP'" + + if [[ -n "$common_configured" ]]; then + test_params="${test_params};common_configured='${common_configured}'" + fi + + PTF_ARGS=(--test-dir "$SAI_TEST_DIR") + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + PTF_ARGS+=(--interface "${port_index}@$(ptf_interface_name "$port_index")") + done + + PTF_ARGS+=(--test-params "$test_params") + PTF_ARGS+=(--xunit --xunit-dir "$xunit_dir") + + # Positive flood tests expect copies on multiple ports, but PTF's flood + # verifier consumes one copy and then rejects the remaining copies via + # verify_no_other_packets(). Relax that final check only for explicitly + # classified flood targets; applying it globally would disable negative + # packet assertions in unrelated tests. + if ptf_target_requires_relax "$test_target"; then + PTF_ARGS+=(--relax) + fi + + if [[ -n "$test_target" ]]; then + PTF_ARGS+=("$test_target") + fi +} + +# Enumerate every test class under $SAI_TEST_DIR as "module.Class" selectors so +# each can run as its own config-reuse ptf invocation. Best-effort: on any +# failure prints nothing and the caller falls back to a single invocation. +enumerate_test_classes() +{ + python3 - "$SAI_TEST_DIR" <<'PY' 2>/dev/null || true +import sys, unittest +test_dir = sys.argv[1] +try: + suite = unittest.TestLoader().discover( + test_dir, pattern="sai_*_test.py", top_level_dir=test_dir) +except Exception: + sys.exit(0) +seen = [] +def walk(s): + for t in s: + if isinstance(t, unittest.TestSuite): + walk(t) + else: + cls = t.__class__ + name = "%s.%s" % (cls.__module__, cls.__name__) + if name not in seen: + seen.append(name) +walk(suite) +for n in seen: + print(n) +PY +} + +print_debug_state() +{ + set +e + log "Linux veth interfaces" + ip -br link show type veth + + if command -v vppctl >/dev/null 2>&1 && timeout "$VPPCTL_TIMEOUT" vppctl show version >/dev/null 2>&1; then + log "VPP interfaces" + timeout "$VPPCTL_TIMEOUT" vppctl show interface + log "VPP hardware interfaces" + timeout "$VPPCTL_TIMEOUT" vppctl show hardware-interfaces + log "Saving VPP API trace to $VPP_API_TRACE" + timeout "$VPPCTL_TIMEOUT" vppctl api trace save "$(basename "$VPP_API_TRACE")" + log "Writing decoded VPP API trace to $VPP_API_TRACE_TXT" + timeout "$VPPCTL_TIMEOUT" vppctl api trace dump > "$VPP_API_TRACE_TXT" 2>&1 + fi + + log "Last 200 lines of $VPP_STDOUT_LOG" + tail -n 200 "$VPP_STDOUT_LOG" 2>/dev/null + log "Last 200 lines of $VPP_LOG" + tail -n 200 "$VPP_LOG" 2>/dev/null + log "Last 200 lines of $SAISERVER_LOG" + tail -n 200 "$SAISERVER_LOG" 2>/dev/null + log "Last 200 lines of $REDIS_LOG" + tail -n 200 "$REDIS_LOG" 2>/dev/null + set -e +} + +terminate_process() +{ + local process_name="$1" + local process_pid="$2" + + if [[ -z "$process_pid" ]]; then + return 0 + fi + + if ! kill -0 "$process_pid" >/dev/null 2>&1; then + wait "$process_pid" 2>/dev/null || true + return 0 + fi + + kill "$process_pid" >/dev/null 2>&1 || true + for ((attempt = 1; attempt <= 5; attempt++)); do + if ! kill -0 "$process_pid" >/dev/null 2>&1; then + wait "$process_pid" 2>/dev/null || true + return 0 + fi + sleep 1 + done + + log "Force stopping $process_name" + kill -9 "$process_pid" >/dev/null 2>&1 || true + wait "$process_pid" 2>/dev/null || true +} + +# In --debug mode, hold the script (PID 1) open after the tests finish so the +# container keeps running with VPP, saiserver, Redis, and the veths still up for +# interactive `vppctl` / `docker exec` inspection. Without this the entrypoint +# would exit right after the last test and Docker would stop the container, +# tearing down exactly the state debug mode is meant to preserve. Exit the +# container with `docker rm -f ` when done. +debug_hold() +{ + [[ "$DEBUG" -eq 1 ]] || return 0 + + log "Debug mode: tests complete; holding container open for inspection." + log "Inspect with: docker exec vppctl show interface" + log "Stop with: docker rm -f " + # Sleep in a loop so the process stays in PID 1 and reaps cleanly on signal. + while true; do + sleep 3600 & + wait "$!" + done +} + +cleanup() +{ + local status="$?" + + set +e + if [[ -n "$KEEP_VETHS_UP_PID" ]]; then + kill "$KEEP_VETHS_UP_PID" >/dev/null 2>&1 || true + fi + if [[ "$status" -ne 0 || "$DEBUG" -eq 1 ]]; then + print_debug_state + fi + + if [[ "$DEBUG" -eq 1 ]]; then + log "Debug mode enabled; leaving VPP, saiserver, and veth interfaces running" + return "$status" + fi + + log "Cleaning up runtime state" + terminate_process saiserver "$SAISERVER_PID" + terminate_process vpp "$VPP_PID" + terminate_process redis "$REDIS_PID" + delete_veths + delete_portchannels + [[ -n "$VPP_CONF" ]] && rm -f "$VPP_CONF" + [[ -n "$VPP_INIT_CLI" ]] && rm -f "$VPP_INIT_CLI" + set -e + + return "$status" +} + +run_ptf() +{ + local -a targets=() + + if [[ "${#TEST_FILTERS[@]}" -gt 0 ]]; then + targets=("${TEST_FILTERS[@]}") + elif [[ -n "$TEST_FILTER" ]]; then + targets=("$TEST_FILTER") + fi + + # Legacy single invocation (reuse disabled): start the backend once and run + # exactly one ptf process for whatever was requested. + if [[ "$COMMON_CONFIGURED_REUSE" != "1" ]]; then + local legacy_target="" + [[ "${#targets[@]}" -gt 0 ]] && legacy_target="${targets[0]}" + start_backend + local legacy_rc=0 + run_one_ptf "$legacy_target" "" "legacy-0" || legacy_rc="$?" + debug_hold + exit "$legacy_rc" + fi + + # Reuse mode. Plan the run as a sequence of (group_id, target) lines, grouped + # by each test's common-config signature (the kwargs it passes to + # T0TestBase.setUp). Tests that need a DIFFERENT common config than the + # group's first test cannot reuse it (e.g. ECMP tests need next-hop groups), + # so each signature gets its own group. Every group runs against a FRESHLY + # restarted backend: the first test in the group builds + persists that + # group's config (common_configured=false), the rest reuse it + # (common_configured=true). The backend restart (fresh saiserver) is what + # makes a second full config build safe — building twice in ONE saiserver + # process crashes it (the original "one test per container" bug). + local plan + if [[ "${#targets[@]}" -gt 0 ]]; then + plan="$(plan_test_groups "${targets[@]}")" + else + log "No test target given; planning all discovered test classes" + plan="$(plan_test_groups)" + fi + + # Per-test isolation: rewrite the group id of every plan line to a unique value + # so each test gets its own freshly restarted backend and rebuilds its own + # common config (common_configured=false). The signature column is preserved for + # logging. This is what lets the upstream OCP tests stay free of config-reuse + # workarounds. + if [[ "$ISOLATE_EACH_TEST" == "1" && -n "$plan" ]]; then + plan="$(printf '%s\n' "$plan" | awk -F'\t' 'NF{printf "%d\t%s\t%s\n", NR-1, $2, $3}')" + log "ISOLATE_EACH_TEST=1: each test runs in its own group (fresh backend + own config)" + fi + + if [[ -z "$plan" ]]; then + log "Planning produced no targets; running full suite in one invocation" + start_backend + local suite_rc=0 + run_one_ptf "" "false" "suite-0" || suite_rc="$?" + debug_hold + exit "$suite_rc" + fi + + local total + total="$(printf '%s\n' "$plan" | grep -c .)" + log "Planned ${total} test target(s) across $(printf '%s\n' "$plan" | cut -f1 | sort -u | grep -c .) config group(s)" + + local overall_rc=0 + local prev_group="" + local idx_in_group=0 + local idx=0 + local group target sig common_configured rc + + while IFS=$'\t' read -r group target sig; do + [[ -z "$target" ]] && continue + idx=$((idx + 1)) + + if [[ "$group" != "$prev_group" ]]; then + # New config group: restart the backend for a clean saiserver, then + # the first test of the group rebuilds the common config. + if [[ -n "$prev_group" ]]; then + stop_backend + fi + log "### Config group ${group} (signature: ${sig:-()}) ###" + start_backend + prev_group="$group" + idx_in_group=0 + fi + + if [[ "$idx_in_group" -eq 0 ]]; then + common_configured="false" + else + common_configured="true" + fi + idx_in_group=$((idx_in_group + 1)) + + log "=== [${idx}/${total}] ${target} (group ${group}, common_configured=${common_configured}) ===" + rc=0 + run_one_ptf "$target" "$common_configured" "group-${group}-test-${idx}" || rc="$?" + if [[ "$rc" -ne 0 ]]; then + if (( rc > overall_rc )); then + overall_rc="$rc" + fi + log "test target '${target}' returned rc=${rc}" + fi + done <<< "$plan" + + debug_hold + exit "$overall_rc" +} + +# Start the runtime backend (Redis + VPP + saiserver + veth-up watchdog). VPP and +# saiserver are fresh processes each time this is called, so a subsequent group's +# common_configured=false config build runs in a clean saiserver and does not hit +# the duplicate-create crash. Veth netdevs persist across backend restarts; +# PortChannel netdevs are created on demand in sai_test setUp (SIMULATE_SONIC). +# across restarts; only the dataplane daemons are recycled. +start_backend() +{ + start_redis + start_vpp + start_saiserver + keep_vpp_veths_up & + KEEP_VETHS_UP_PID="$!" +} + +# Stop the runtime backend and reset per-run state so the next group starts clean. +# +# NOTE: ideally each test would restore the initial state via SAI teardown in its +# own tearDown(), avoiding this full backend recycle. That does not fully work yet +# because the VS-VPP backend's object removal is not idempotent enough to rebuild +# the whole T0 config a second time in one saiserver process: on remove, leftover +# VPP state (linux-cp pairs, bridge domains, bonds, etc.) is not always torn down, +# so the next create hits "already exists" (VPP VALUE_EXIST/-81) and fails. The +# host-interface case was fixed in sonic-sairedis PR #1952 (vs_remove_hostif now +# deletes the linux-cp pair + disables IPv6, and create tolerates VALUE_EXIST), but +# VLAN, bridge-port, LAG, RIF and route removal still need the same treatment. +# Until that is done, we restart the backend (fresh saiserver) per group so each +# config build starts from clean VPP state. See devdocs/progress-7-3-hostif-removal.md. +stop_backend() +{ + if [[ -n "$KEEP_VETHS_UP_PID" ]]; then + kill "$KEEP_VETHS_UP_PID" >/dev/null 2>&1 || true + wait "$KEEP_VETHS_UP_PID" 2>/dev/null || true + KEEP_VETHS_UP_PID="" + fi + terminate_process saiserver "$SAISERVER_PID"; SAISERVER_PID="" + terminate_process vpp "$VPP_PID"; VPP_PID="" + terminate_process redis "$REDIS_PID"; REDIS_PID="" + # Drop persisted SAI object IDs and the link-up marker so the next group's + # first test rebuilds (and re-persists) its own config and re-asserts veths. + rm -rf /tmp/sai_model 2>/dev/null || true + rm -f "$LINKS_UP_MARKER" 2>/dev/null || true +} + +# Emit a run plan: tab-separated "\t\t" lines, +# grouped by each test class's common-config signature so identical-config tests +# run together (and can reuse one config build). With no args, plans every +# discovered test class under $SAI_TEST_DIR. +plan_test_groups() +{ + SAI_TEST_DIR="$SAI_TEST_DIR" python3 - "$@" <<'PY' +import ast, os, sys, glob, collections + +test_dir = os.environ.get("SAI_TEST_DIR", "/sai_test") +targets = sys.argv[1:] + +# kwargs that do NOT change the common config (so they must not split groups) +NON_CONFIG_KW = {"skip_reason", "wait_sec"} + +mod_file = {} +for p in glob.glob(os.path.join(test_dir, "sai_*_test.py")): + mod_file[os.path.basename(p)[:-3]] = p + +# Build a cross-file class registry: name -> {bases:[...], setup:FunctionDef|None, +# module:str}. Classes can subclass intermediate test bases (e.g. EcmpBaseTestV4) +# whose setUp sets the common-config kwargs, so signatures must resolve through +# the inheritance chain, not just the class's own setUp. +registry = {} +mod_classes = collections.OrderedDict() +for mod in sorted(mod_file): + try: + tree = ast.parse(open(mod_file[mod]).read(), mod_file[mod]) + except Exception: + continue + names = [] + for node in tree.body: + if not isinstance(node, ast.ClassDef): + continue + names.append(node.name) + bases = [b.id for b in node.bases if isinstance(b, ast.Name)] + setup = next((m for m in node.body + if isinstance(m, ast.FunctionDef) and m.name == "setUp"), None) + # last definition wins if duplicated + registry[node.name] = {"bases": bases, "setup": setup, "module": mod} + mod_classes[mod] = names + +def _kwargs_from_setup_call(setup): + """Return (kwargs_str_or_None, base_to_recurse_or_None) for a class's setUp. + kwargs_str: config signature if this setUp passes config kwargs. + base_to_recurse: if setUp only chains to super()/Base.setUp() without config + kwargs, the class name to resolve the signature from.""" + for sub in ast.walk(setup): + if not (isinstance(sub, ast.Call) and isinstance(sub.func, ast.Attribute) + and sub.func.attr == "setUp"): + continue + kws = [] + for kw in sub.keywords: + if kw.arg in NON_CONFIG_KW or kw.arg is None: + continue + try: + val = ast.literal_eval(kw.value) + except Exception: + val = "?" + kws.append("%s=%s" % (kw.arg, val)) + if kws: + return ("|".join(sorted(kws)), None) + # No config kwargs: figure out which base this chains to. + f = sub.func.value + if isinstance(f, ast.Call) and isinstance(f.func, ast.Name) \ + and f.func.id == "super": + return (None, "__super__") # super().setUp() + if isinstance(f, ast.Name): + return (None, f.id) # .setUp(self, ...) + return (None, "__super__") + return ("()", None) + +def signature(clsname, seen=None): + if seen is None: + seen = set() + if clsname in seen or clsname not in registry: + return "()" # unknown base (e.g. T0TestBase) -> default config + seen.add(clsname) + info = registry[clsname] + setup = info["setup"] + if setup is None: + # inherits setUp wholesale from first base + for b in info["bases"]: + return signature(b, seen) + return "()" + sig, base = _kwargs_from_setup_call(setup) + if sig is not None: + return sig + # chained to super/base without config kwargs -> resolve that base + if base == "__super__": + for b in info["bases"]: + return signature(b, seen) + return "()" + return signature(base, seen) + +# Expand requested targets (or everything) into (module, class) pairs. +pairs = [] +if not targets: + for mod in mod_classes: + for c in mod_classes[mod]: + pairs.append((mod, c)) +else: + for t in targets: + if "." in t: + mod, cls = t.split(".", 1) + pairs.append((mod, cls)) + elif t in mod_classes: + for c in mod_classes[t]: + pairs.append((t, c)) + else: + pairs.append((t, "")) # unknown selector: run as-is, default group + +# Group by resolved signature, preserving first-seen order. +groups = collections.OrderedDict() +for mod, cls in pairs: + sig = signature(cls) if cls else "()" + groups.setdefault(sig, []).append("%s.%s" % (mod, cls) if cls else mod) + +for gid, (sig, tlist) in enumerate(groups.items()): + for t in tlist: + sys.stdout.write("%d\t%s\t%s\n" % (gid, t, sig)) +PY +} + +# Copy one PTF invocation's JUnit XML into the shared results directory without +# replacing an earlier invocation's same-named TEST-.xml report. +collect_xunit_results() +{ + local xunit_dir="$1" + local invocation_namespace="$2" + local xml_file + local xml_name + local namespaced_name + local -a xml_files + + [[ -n "$TEST_RESULTS_DIR" ]] || return 0 + + mkdir -p "$TEST_RESULTS_DIR" + shopt -s nullglob + xml_files=("$xunit_dir"/*.xml) + shopt -u nullglob + + for xml_file in "${xml_files[@]}"; do + xml_name="$(basename "$xml_file")" + if [[ "$xml_name" == TEST-* ]]; then + namespaced_name="TEST-${invocation_namespace}-${xml_name#TEST-}" + else + namespaced_name="TEST-${invocation_namespace}-${xml_name}" + fi + cp "$xml_file" "$TEST_RESULTS_DIR/$namespaced_name" + done +} + +# Run a single ptf invocation for one target. Echoes PTF output through and +# triggers the one-time veth bring-up on the framework's "Turn up ports..." +# marker. Returns ptf's exit code. +# +# PTF rmtree's its --xunit-dir on startup, so each invocation gets its own +# private temp dir (which PTF may freely wipe), and the resulting JUnit XML is +# copied into the shared $TEST_RESULTS_DIR afterward. This keeps results from +# accumulating across invocations and avoids EBUSY when $TEST_RESULTS_DIR is a +# bind mount (rmtree of a mount point fails). +run_one_ptf() +{ + local test_target="$1" + local common_configured="$2" + local invocation_namespace="$3" + local test_rc + local xunit_dir + + xunit_dir="$(mktemp -d /tmp/ptf-xunit.XXXXXX)" + + build_ptf_args "$test_target" "$common_configured" "$xunit_dir" + log "Running PTF${test_target:+ filter: $test_target}" + + set +e + PYTHONUNBUFFERED=1 ptf "${PTF_ARGS[@]}" 2>&1 | while IFS= read -r ptf_line; do + printf '%s\n' "$ptf_line" + + case "$ptf_line" in + *"Turn up ports..."*) + bring_up_veths + ;; + esac + done + test_rc="${PIPESTATUS[0]}" + set -e + + collect_xunit_results "$xunit_dir" "$invocation_namespace" + rm -rf "$xunit_dir" + + return "$test_rc" +} + +main() +{ + exec_requested_shell "$@" + parse_args "$@" + trap cleanup EXIT + + preflight + disable_ipv6_autoconf + create_veths + create_sonic_vpp_ifmap + run_ptf +} + +main "$@" \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/sai.profile b/.azure-pipelines/docker-sai-test-vpp/sai.profile new file mode 100644 index 0000000000..d8de1b9d5b --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/sai.profile @@ -0,0 +1,3 @@ +SAI_VS_SWITCH_TYPE=SAI_VS_SWITCH_TYPE_VPP +SAI_VS_HOSTIF_USE_TAP_DEVICE=true +SAI_VS_INTERFACE_LANE_MAP_FILE=/etc/sai/lanemap.ini \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/swss_log_stdout_preload.cpp b/.azure-pipelines/docker-sai-test-vpp/swss_log_stdout_preload.cpp new file mode 100644 index 0000000000..57425561de --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/swss_log_stdout_preload.cpp @@ -0,0 +1,18 @@ +// Harness-only LD_PRELOAD shim: route SAI VS library SWSS_LOG_* to stdout so +// run_test.sh capture in /var/log/sai-server.log works without pulling SONiC +// logger setup back into the SAI server binary. +#include + +namespace { + +__attribute__((constructor)) +static void route_swss_log_to_stdout() +{ + SWSS_LOG_ENTER(); + swss::Logger::setMinPrio(swss::Logger::SWSS_DEBUG); + // Standard error keeps function entry and exit logs out of captured standard + // output. The harness redirects both streams to the SAI server log. + swss::Logger::swssOutputNotify("saiserver", "STDERR"); +} + +} // namespace diff --git a/.azure-pipelines/docker-sai-test-vpp/test_derive_ptf_version.py b/.azure-pipelines/docker-sai-test-vpp/test_derive_ptf_version.py new file mode 100644 index 0000000000..d1ef8df371 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/test_derive_ptf_version.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import importlib.util +import os +import tempfile +import unittest +from unittest import mock + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +MODULE_PATH = os.path.join(SCRIPT_DIR, "derive_ptf_version.py") +SPEC = importlib.util.spec_from_file_location("derive_ptf_version", MODULE_PATH) +VERSION = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(VERSION) + + +class DerivePtfVersionTest(unittest.TestCase): + def test_release_tag(self): + self.assertEqual("0.12.1", VERSION.version_from_describe("v0.12.1-0-gabc1234")) + + def test_post_release(self): + self.assertEqual( + "0.12.1.post7+gd587084", + VERSION.version_from_describe("v0.12.1-7-gd587084"), + ) + + def test_invalid_description(self): + with self.assertRaisesRegex(ValueError, "unsupported git describe"): + VERSION.version_from_describe("d587084") + + def test_version_file_fallback(self): + with tempfile.TemporaryDirectory() as temp_dir: + with open(os.path.join(temp_dir, "Version.txt"), "w", encoding="utf-8") as version_file: + version_file.write("0.12.1\n") + self.assertEqual( + "0.12.1+gd587084", + VERSION.fallback_version(temp_dir, "d587084"), + ) + + def test_untagged_fallback(self): + with tempfile.TemporaryDirectory() as temp_dir: + with mock.patch.object(VERSION, "git", return_value="185"): + self.assertEqual( + "0.0.post185+g4ee4c6a", + VERSION.fallback_version(temp_dir, "4ee4c6a"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.azure-pipelines/docker-sai-test-vpp/test_evaluate_ci_baseline.py b/.azure-pipelines/docker-sai-test-vpp/test_evaluate_ci_baseline.py new file mode 100644 index 0000000000..981abeea32 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/test_evaluate_ci_baseline.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import importlib.util +import os +import tempfile +import unittest + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +MODULE_PATH = os.path.join(SCRIPT_DIR, "evaluate_ci_baseline.py") +SPEC = importlib.util.spec_from_file_location("evaluate_ci_baseline", MODULE_PATH) +BASELINE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BASELINE) + + +class EvaluateCiBaselineTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.xml_dir = os.path.join(self.temp_dir.name, "xml") + os.makedirs(self.xml_dir) + + def write_result(self, selector, result="PASS"): + result_xml = { + "PASS": "", + "FAIL": '', + "ERROR": '', + "SKIP": '', + }[result] + xml = ( + '' + f'' + f"{result_xml}" + ) + filename = f"TEST-{selector}.xml" + with open(os.path.join(self.xml_dir, filename), "w", encoding="utf-8") as xml_file: + xml_file.write(xml) + + def write_selectors(self, filename, selectors): + path = os.path.join(self.temp_dir.name, filename) + with open(path, "w", encoding="utf-8") as selector_file: + selector_file.write("\n".join(selectors) + "\n") + return path + + def test_stable_pass_with_known_failure_and_candidate(self): + self.write_result("module.StableTest") + self.write_result("module.KnownFailure", "FAIL") + self.write_result("module.NewPass") + + results = BASELINE.collect_results(self.xml_dir) + report, regressions = BASELINE.format_report( + {"module.StableTest"}, results, matrix_rc=1 + ) + + self.assertEqual([], regressions) + self.assertIn("New pass candidates: 1", report) + self.assertIn("Known non-baseline non-passes: 1", report) + + def test_regressed_and_missing_baseline_tests_fail(self): + self.write_result("module.RegressedTest", "ERROR") + + results = BASELINE.collect_results(self.xml_dir) + _, regressions = BASELINE.format_report( + {"module.RegressedTest", "module.MissingTest"}, results, matrix_rc=1 + ) + + self.assertEqual( + [("module.MissingTest", "MISSING"), ("module.RegressedTest", "ERROR")], + regressions, + ) + + def test_malformed_junit_is_infrastructure_error(self): + path = os.path.join(self.xml_dir, "TEST-broken.xml") + with open(path, "w", encoding="utf-8") as xml_file: + xml_file.write("") + + with self.assertRaisesRegex(ValueError, "cannot parse"): + BASELINE.collect_results(self.xml_dir) + + def test_duplicate_baseline_selector_is_rejected(self): + path = self.write_selectors("baseline.txt", ["module.Test", "module.Test"]) + + with self.assertRaisesRegex(ValueError, "duplicate baseline selector"): + BASELINE.read_baseline(path) + + def test_incomplete_matrix_is_rejected(self): + self.write_result("module.StableTest") + results = BASELINE.collect_results(self.xml_dir) + + error = BASELINE.validate_matrix_contract( + {"module.StableTest", "module.ExpectedTest"}, results + ) + + self.assertIn("module.ExpectedTest", error) + + def test_infrastructure_matrix_status_fails_main(self): + baseline = self.write_selectors("baseline.txt", ["module.StableTest"]) + expected = self.write_selectors("expected.txt", ["module.StableTest"]) + report = os.path.join(self.temp_dir.name, "report.txt") + + rc = BASELINE.main([ + "--xml-dir", self.xml_dir, + "--baseline", baseline, + "--expected", expected, + "--matrix-rc", "2", + "--report", report, + ]) + + self.assertEqual(1, rc) + with open(report, encoding="utf-8") as report_file: + self.assertIn("Infrastructure failure", report_file.read()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.azure-pipelines/docker-sai-test-vpp/vpp_startup.conf.template b/.azure-pipelines/docker-sai-test-vpp/vpp_startup.conf.template new file mode 100644 index 0000000000..354e850b13 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/vpp_startup.conf.template @@ -0,0 +1,57 @@ +unix { + nodaemon + log __VPP_LOG__ + full-coredump + cli-listen /run/vpp/cli.sock + poll-sleep-usec 100 + exec __VPP_INIT_CLI__ +} + +api-trace { + on + nitems 32768 +} + +api-segment { + global-size 256M + api-size 64M +} + +socksvr { + default +} + +memory { + main-heap-size 4G +} + +l3fib { + fib-entry-pool-size 256K + load-balance-pool-size 256K + ip4-mtrie-pool-size 256K +} + +ip6 { + heap-size 128M +} + +plugins { + plugin default { disable } + plugin af_packet_plugin.so { enable } + plugin tap_plugin.so { enable } + plugin linux_cp_plugin.so { enable } + plugin linux_nl_plugin.so { enable } + plugin acl_plugin.so { enable } + plugin vxlan_plugin.so { enable } + plugin tunterm_acl_plugin.so { enable } + plugin ip_validate_plugin.so { enable } + plugin sflow_plugin.so { enable } +} + +linux-cp { + lcp-auto-subint +} + +buffers { + buffers-per-numa __BUFFERS_PER_NUMA__ +} diff --git a/.azure-pipelines/test-sai-vpp-template.yml b/.azure-pipelines/test-sai-vpp-template.yml new file mode 100644 index 0000000000..8c7a6af260 --- /dev/null +++ b/.azure-pipelines/test-sai-vpp-template.yml @@ -0,0 +1,127 @@ +parameters: +- name: timeout + type: number + default: 360 + +- name: image_artifact_name + type: string + +- name: log_artifact_name + type: string + +jobs: +- job: RunSaiVppMatrix + displayName: Run VPP SAI compatibility tests + timeoutInMinutes: ${{ parameters.timeout }} + + pool: sonictest + + steps: + - checkout: self + clean: true + displayName: Checkout sonic-sairedis + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: ${{ parameters.image_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/download + displayName: Download pre-stage built VPP SAI test image + + - script: | + set -u + mapfile -t stale_containers < <(sudo docker ps -aq --filter label=com.sonic.saivpp-ci=true) + if [[ "${#stale_containers[@]}" -gt 0 ]]; then + sudo docker rm -f "${stale_containers[@]}" + fi + mapfile -t stale_images < <(sudo docker image ls -q --filter label=com.sonic.saivpp-ci=true | sort -u) + if [[ "${#stale_images[@]}" -gt 0 ]]; then + sudo docker image rm -f "${stale_images[@]}" + fi + displayName: Clean stale VPP SAI test resources + + - script: | + set -euxo pipefail + + download_dir="$(Build.ArtifactStagingDirectory)/download" + results_dir="$(Build.ArtifactStagingDirectory)/sai-vpp-results" + xml_dir="$results_dir/xml" + log_dir="$results_dir/log" + image_tag="$(cat "$download_dir/image-tag.txt")" + + mkdir -p "$xml_dir" "$log_dir" + chmod 0777 "$xml_dir" "$log_dir" + sudo docker load -i "$download_dir/docker-sai-test-vpp.gz" + sudo docker image inspect "$image_tag" > "$results_dir/image-inspect.json" + + set +e + sudo docker run --rm --privileged \ + --name "saivpp-ci-$(Build.BuildId)-$(System.JobAttempt)" \ + --label com.sonic.saivpp-ci=true \ + -e PORT_COUNT=32 \ + -e ISOLATE_EACH_TEST=1 \ + -e STARTUP_TIMEOUT=180 \ + -v "$xml_dir:/test-results" \ + -v "$log_dir:/var/log" \ + "$image_tag" \ + sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test \ + 2>&1 | tee "$results_dir/run.log" + matrix_rc=${PIPESTATUS[0]} + set -e + + echo "$matrix_rc" > "$results_dir/matrix.rc" + sudo chown -R "$(id -u):$(id -g)" "$results_dir" + chmod -R a+rX "$results_dir" + echo "VPP SAI matrix exit code: $matrix_rc" + displayName: Run VPP SAI full matrix + + - script: | + set -euxo pipefail + + results_dir="$(Build.ArtifactStagingDirectory)/sai-vpp-results" + harness_dir="$(System.DefaultWorkingDirectory)/.azure-pipelines/docker-sai-test-vpp" + matrix_rc="$(cat "$results_dir/matrix.rc")" + + python3 -c 'import defusedxml' 2>/dev/null || sudo uv pip install --system defusedxml + python3 "$harness_dir/gen_compatibility_matrix.py" \ + "$results_dir/xml" "$results_dir/compatibility-matrix.md" + python3 "$harness_dir/evaluate_ci_baseline.py" \ + --xml-dir "$results_dir/xml" \ + --baseline "$harness_dir/ci-pass-tests.txt" \ + --expected "$harness_dir/ci-matrix-tests.txt" \ + --matrix-rc "$matrix_rc" \ + --report "$results_dir/baseline-report.txt" + displayName: Evaluate VPP SAI stable baseline + + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + searchFolder: $(Build.ArtifactStagingDirectory)/sai-vpp-results/xml + testResultsFiles: 'TEST-*.xml' + testRunTitle: VPP SAI PTF + failTaskOnFailedTests: false + failTaskOnMissingResultsFile: true + condition: always() + + - script: | + set -euxo pipefail + cp -v "$(Build.ArtifactStagingDirectory)/download/provenance.txt" \ + "$(Build.ArtifactStagingDirectory)/sai-vpp-results/" + cp -v "$(Build.ArtifactStagingDirectory)/download/SHA256SUMS" \ + "$(Build.ArtifactStagingDirectory)/sai-vpp-results/" + displayName: Collect VPP SAI provenance + condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/sai-vpp-results + artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt) + displayName: Publish VPP SAI logs and results + condition: always() + + - script: | + set -u + tag_file="$(Build.ArtifactStagingDirectory)/download/image-tag.txt" + if [[ -f "$tag_file" ]]; then + image_tag="$(cat "$tag_file")" + sudo docker image rm "$image_tag" || true + fi + displayName: Clean VPP SAI test image + condition: always() diff --git a/.gitignore b/.gitignore index db4a306321..ea79bc7d57 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ # Packaging Files # ################### +/*.buildinfo +/*.changes +/*.deb debian/*.debhelper.log debian/*.substvars debian/.debhelper/ @@ -18,6 +21,7 @@ debian/autoreconf.after debian/autoreconf.before debian/debhelper-build-stamp debian/files +.azure-pipelines/docker-sai-test-vpp/debs/ debian/python-pysairedis/ debian/python3-pysairedis/ diff --git a/SAI b/SAI index c67f115230..07144ed12f 160000 --- a/SAI +++ b/SAI @@ -1 +1 @@ -Subproject commit c67f1152309ca08a94de0be8634a733c5cb25c35 +Subproject commit 07144ed12f13b4d621a0d3322a955948d5b26f50 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2f903223fd..2312b50b30 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,6 +38,9 @@ parameters: - name: debian_version type: string default: bookworm + - name: vpp_run_id + type: string + default: '' variables: - name: BUILD_BRANCH ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: @@ -76,8 +79,7 @@ stages: debian_version: ${{ parameters.debian_version }} - stage: BuildArm - dependsOn: Build - condition: succeeded('Build') + dependsOn: [] jobs: - template: .azure-pipelines/build-template.yml parameters: @@ -100,17 +102,57 @@ stages: debian_version: ${{ parameters.debian_version }} - stage: BuildTrixie - dependsOn: BuildArm - condition: succeeded('BuildArm') + dependsOn: [] jobs: + - job: ResolveVpp + displayName: Resolve VPP build + pool: + vmImage: 'ubuntu-22.04' + steps: + - ${{ if eq(parameters.vpp_run_id, '') }}: + - task: DownloadPipelineArtifact@2 + name: downloadVpp + inputs: + source: specific + project: build + pipeline: sonic-net.sonic-platform-vpp + artifact: vpp-trixie + path: $(Build.ArtifactStagingDirectory)/vpp-resolution + patterns: '**/libvppinfra_*_amd64.deb' + runVersion: latestFromBranch + runBranch: refs/heads/master + allowPartiallySucceededBuilds: true + displayName: Resolve latest successful sonic platform-vpp build + - ${{ if eq(parameters.vpp_run_id, '') }}: + - script: | + set -euo pipefail + vpp_run_id="$(downloadVpp.BuildNumber)" + test -n "$vpp_run_id" + echo "Resolved sonic platform-vpp run ID: $vpp_run_id" + echo "##vso[task.setvariable variable=VPP_RUN_ID;isOutput=true]$vpp_run_id" + name: resolveVppRun + displayName: Publish resolved VPP run ID + - ${{ if ne(parameters.vpp_run_id, '') }}: + - script: | + set -euo pipefail + vpp_run_id="${{ parameters.vpp_run_id }}" + test -n "$vpp_run_id" + echo "Using requested sonic platform-vpp run ID: $vpp_run_id" + echo "##vso[task.setvariable variable=VPP_RUN_ID;isOutput=true]$vpp_run_id" + name: resolveVppRun + displayName: Publish requested VPP run ID + - template: .azure-pipelines/build-template.yml parameters: arch: amd64 + depends_on: ResolveVpp swss_common_artifact_name: sonic-swss-common-trixie artifact_name: sonic-sairedis-trixie syslog_artifact_name: sonic-sairedis-trixie.syslog run_unit_test: true + saithrift_v2: true debian_version: trixie + vpp_run_id: $(VPP_RUN_ID) - template: .azure-pipelines/build-template.yml parameters: @@ -132,6 +174,29 @@ stages: syslog_artifact_name: sonic-sairedis-trixie.syslog.arm64 debian_version: trixie +- stage: BuildSaiTestVpp + dependsOn: BuildTrixie + condition: succeeded('BuildTrixie') + variables: + VPP_RUN_ID: $[ stageDependencies.BuildTrixie.ResolveVpp.outputs['resolveVppRun.VPP_RUN_ID'] ] + jobs: + - template: .azure-pipelines/build-docker-sai-test-vpp-template.yml + parameters: + timeout: 90 + sairedis_artifact_name: sonic-sairedis-trixie + swss_common_artifact_name: sonic-swss-common-trixie + artifact_name: docker-sai-test-vpp + vpp_run_id: $(VPP_RUN_ID) + +- stage: TestSaiVpp + dependsOn: BuildSaiTestVpp + condition: succeeded('BuildSaiTestVpp') + jobs: + - template: .azure-pipelines/test-sai-vpp-template.yml + parameters: + image_artifact_name: docker-sai-test-vpp + log_artifact_name: sai-vpp-test-results + - stage: BuildSwss dependsOn: Build condition: succeeded('Build') diff --git a/pyext/py3/Makefile.am b/pyext/py3/Makefile.am index 9259566860..86090f9181 100644 --- a/pyext/py3/Makefile.am +++ b/pyext/py3/Makefile.am @@ -10,7 +10,8 @@ BUILT_SOURCES = pysairedis_wrap.cpp _pysairedis_la_SOURCES = pysairedis_wrap.cpp $(SOURCES) _pysairedis_la_CXXFLAGS = $(PYTHON3_CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS_COMMON) \ - -Wno-cast-align -Wno-cast-qual -Wno-shadow -Wno-redundant-decls + -Wno-cast-align -Wno-cast-qual -Wno-shadow -Wno-redundant-decls \ + -Wno-error=disabled-optimization _pysairedis_la_LDFLAGS = -module \ -lhiredis -lswsscommon -lpthread \ diff --git a/syncd/SwitchNotifications.h b/syncd/SwitchNotifications.h index a63a723ce5..7ad89395d3 100644 --- a/syncd/SwitchNotifications.h +++ b/syncd/SwitchNotifications.h @@ -169,6 +169,7 @@ namespace syncd .on_ipsec_post_status = &Slot::onIpsecPostStatus, .on_switch_macsec_post_status = &Slot::onSwitchMacsecPostStatus, .on_switch_ipsec_post_status = &Slot::onSwitchIpsecPostStatus, + .on_next_hop_group_hw_protection_switchover = nullptr, .on_ha_set_event = &Slot::onHaSetEvent, .on_ha_scope_event = &Slot::onHaScopeEvent, .on_flow_bulk_get_session_event = &Slot::onFlowBulkGetSessionEvent, diff --git a/unittest/meta/TestSaiSerialize.cpp b/unittest/meta/TestSaiSerialize.cpp index 67dd8e61f2..a79394e163 100644 --- a/unittest/meta/TestSaiSerialize.cpp +++ b/unittest/meta/TestSaiSerialize.cpp @@ -1299,8 +1299,8 @@ TEST(SaiSerialize, serialize_qos_map) attr.id = SAI_QOS_MAP_ATTR_MAP_TO_VALUE_LIST; sai_qos_map_t qm = { - .key = { .tc = 1, .dscp = 2, .dot1p = 3, .prio = 4, .pg = 5, .queue_index = 6, .color = SAI_PACKET_COLOR_RED, .mpls_exp = 0, .fc = 2 }, - .value = { .tc = 11, .dscp = 22, .dot1p = 33, .prio = 44, .pg = 55, .queue_index = 66, .color = SAI_PACKET_COLOR_GREEN, .mpls_exp = 0, .fc = 2 } }; + .key = { .tc = 1, .dscp = 2, .dot1p = 3, .prio = 4, .pg = 5, .queue_index = 6, .color = SAI_PACKET_COLOR_RED, .mpls_exp = 0, .fc = 2, .dei = 0, .vc = 0 }, + .value = { .tc = 11, .dscp = 22, .dot1p = 33, .prio = 44, .pg = 55, .queue_index = 66, .color = SAI_PACKET_COLOR_GREEN, .mpls_exp = 0, .fc = 2, .dei = 0, .vc = 0 } }; attr.value.qosmap.count = 1; attr.value.qosmap.list = &qm; diff --git a/unittest/syncd/TestAttrVersionChecker.cpp b/unittest/syncd/TestAttrVersionChecker.cpp index f1bc91ab94..493b6566de 100644 --- a/unittest/syncd/TestAttrVersionChecker.cpp +++ b/unittest/syncd/TestAttrVersionChecker.cpp @@ -93,6 +93,7 @@ TEST(AttrVersionChecker, reset) .iscustom = false,\ .apiversion = (v),\ .nextrelease = (n),\ + .valueprecision = 0,\ };\ diff --git a/vslib/vpp/SwitchVpp.cpp b/vslib/vpp/SwitchVpp.cpp index 890c6e8b80..17ea7fa3a3 100644 --- a/vslib/vpp/SwitchVpp.cpp +++ b/vslib/vpp/SwitchVpp.cpp @@ -14,6 +14,7 @@ #include #include +#include #include using namespace saivs; @@ -152,6 +153,11 @@ sai_status_t SwitchVpp::create_qos_queues_per_port( attr.value.oid = port_id; CHECK_STATUS(set(SAI_OBJECT_TYPE_QUEUE, queue_id, &attr)); + + attr.id = SAI_QUEUE_ATTR_PARENT_SCHEDULER_NODE; + attr.value.oid = SAI_NULL_OBJECT_ID; + + CHECK_STATUS(set(SAI_OBJECT_TYPE_QUEUE, queue_id, &attr)); } attr.id = SAI_PORT_ATTR_QOS_NUMBER_OF_QUEUES; @@ -202,6 +208,11 @@ sai_status_t SwitchVpp::create_cpu_qos_queues( attr.value.oid = port_id; CHECK_STATUS(set(SAI_OBJECT_TYPE_QUEUE, queue_id, &attr)); + + attr.id = SAI_QUEUE_ATTR_PARENT_SCHEDULER_NODE; + attr.value.oid = SAI_NULL_OBJECT_ID; + + CHECK_STATUS(set(SAI_OBJECT_TYPE_QUEUE, queue_id, &attr)); } attr.id = SAI_PORT_ATTR_QOS_NUMBER_OF_QUEUES; @@ -741,6 +752,39 @@ bool SwitchVpp::port_to_hostif_list( return getTapNameFromPortId(port_id, if_name); } +bool SwitchVpp::getTapNameFromPortOrLagId( + _In_ sai_object_id_t obj_id, + _Out_ std::string& if_name) +{ + SWSS_LOG_ENTER(); + + sai_object_type_t ot = objectTypeQuery(obj_id); + + if (ot == SAI_OBJECT_TYPE_PORT) + { + return getTapNameFromPortId(obj_id, if_name); + } + + if (ot == SAI_OBJECT_TYPE_LAG) + { + platform_bond_info_t bond_info; + sai_status_t status = get_lag_bond_info(obj_id, bond_info); + + if (status != SAI_STATUS_SUCCESS) + { + return false; + } + + std::ostringstream tap_stream; + tap_stream << "be" << bond_info.id; + if_name = tap_stream.str(); + + return true; + } + + return false; +} + bool SwitchVpp::port_to_hwifname( _In_ sai_object_id_t port_id, _Inout_ std::string& if_name) @@ -1303,6 +1347,8 @@ sai_status_t SwitchVpp::create( { SWSS_LOG_ENTER(); + serviceDeferredOperStatusResync(); + if (object_type == SAI_OBJECT_TYPE_DEBUG_COUNTER) { sai_object_id_t object_id; @@ -1658,6 +1704,8 @@ sai_status_t SwitchVpp::remove( { SWSS_LOG_ENTER(); + serviceDeferredOperStatusResync(); + if (object_type == SAI_OBJECT_TYPE_DEBUG_COUNTER) { sai_object_id_t objectId; @@ -2017,6 +2065,8 @@ sai_status_t SwitchVpp::set( { SWSS_LOG_ENTER(); + serviceDeferredOperStatusResync(); + if (objectType == SAI_OBJECT_TYPE_PORT) { sai_object_id_t objectId; diff --git a/vslib/vpp/SwitchVpp.h b/vslib/vpp/SwitchVpp.h index 26eab9f10e..d9c40dc076 100644 --- a/vslib/vpp/SwitchVpp.h +++ b/vslib/vpp/SwitchVpp.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -724,7 +725,9 @@ namespace saivs _In_ const std::string &serializedObjectId, _In_ uint32_t attr_count, _In_ const sai_attribute_t *attr_list, - _In_ bool is_add); + _In_ bool is_add, + _In_ bool program_adjacency = true, + _In_ bool program_host_route = false); sai_status_t addIpNbr( _In_ const std::string &serializedObjectId, @@ -775,6 +778,10 @@ namespace saivs _In_ bool is_add, _Out_ uint32_t *stats_index = nullptr); + const char* resolveNexthopMemberHwif( + _In_ const nexthop_grp_member_t *member, + _Out_ std::string &member_hwif); + sai_status_t updateIpRoute( _In_ const std::string &serializedObjectId, _In_ const sai_attribute_t *attr_list); @@ -1145,6 +1152,10 @@ namespace saivs void populate_if_mapping(); + bool getTapNameFromPortOrLagId( + _In_ sai_object_id_t obj_id, + _Out_ std::string& if_name); + const char *tap_to_hwif_name(const char *name); const char *hwif_to_tap_name(const char *name); @@ -1155,6 +1166,17 @@ namespace saivs void vppProcessEvents (); + void resyncPortOperStatus(); + + // Run a deferred oper-status resync on the command thread if the + // event thread has requested one. resyncPortOperStatus() issues VPP + // binary-API calls (interface_get_state) which allocate on VPP's + // non-thread-safe clib heap; running them on the background event + // thread races the command thread's clib allocations and crashes + // (os_panic in clib_mem_heap_realloc_aligned). So the event thread + // only flags that a resync is due and the command thread performs it. + void serviceDeferredOperStatusResync(); + void startVppEventsThread(); private: // VPP @@ -1163,6 +1185,7 @@ namespace saivs std::map m_hwif_hostif_map; int mapping_init = 0; bool m_run_vpp_events_thread = true; + std::atomic m_operResyncDue { false }; bool VppEventsThreadStarted = false; std::shared_ptr m_vpp_thread; @@ -1251,6 +1274,8 @@ namespace saivs std::map> m_hostif_info_map; + std::map m_last_oper_up; + CRMTracker m_crmTracker; bool isIPv4Route(const std::string &serializedObjectId); diff --git a/vslib/vpp/SwitchVppFdb.cpp b/vslib/vpp/SwitchVppFdb.cpp index d4d79d8bba..a85a108d3f 100644 --- a/vslib/vpp/SwitchVppFdb.cpp +++ b/vslib/vpp/SwitchVppFdb.cpp @@ -445,15 +445,40 @@ sai_status_t SwitchVpp::vpp_create_bvi_interface( return SAI_STATUS_FAILURE; } + sai_mac_t mac_addr; + auto attr_mac_addr = sai_metadata_get_attr_by_id(SAI_ROUTER_INTERFACE_ATTR_SRC_MAC_ADDRESS, attr_count, attr_list); - if (attr_mac_addr == NULL) + if (attr_mac_addr != NULL) { - SWSS_LOG_NOTICE("attr ROUTER INTERFACE MAC Address is not found"); - return SAI_STATUS_FAILURE; + memcpy(mac_addr, attr_mac_addr->value.mac, sizeof(sai_mac_t)); } + else + { + // SAI_ROUTER_INTERFACE_ATTR_SRC_MAC_ADDRESS is optional; per the SAI + // spec its default is the switch source MAC. When a caller omits it + // (e.g. the OCP sai_test PTF suite creates VLAN RIFs without a MAC, + // unlike orchagent which always supplies the switch MAC), fall back to + // SAI_SWITCH_ATTR_SRC_MAC_ADDRESS instead of failing the BVI create. + // Without this the VLAN BVI is never created, so SVI ingress traffic + // floods L2 instead of being routed - which broke every standalone L3 + // route/RIF test whose ingress is a VLAN access port. See + // .azure-pipelines/docker-sai-test-vpp/devdocs/progress-6-17.md (Issue B). + sai_attribute_t sw_attr; + memset(&sw_attr, 0, sizeof(sw_attr)); + sw_attr.id = SAI_SWITCH_ATTR_SRC_MAC_ADDRESS; + + sai_status_t mac_status = get(SAI_OBJECT_TYPE_SWITCH, m_switch_id, 1, &sw_attr); + if (mac_status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("RIF src MAC not provided and failed to read switch src MAC on %s: %s", + sai_serialize_object_id(m_switch_id).c_str(), + sai_serialize_status(mac_status).c_str()); + return SAI_STATUS_FAILURE; + } - sai_mac_t mac_addr; - memcpy(mac_addr, attr_mac_addr->value.mac, sizeof(sai_mac_t)); + memcpy(mac_addr, sw_attr.value.mac, sizeof(sai_mac_t)); + SWSS_LOG_NOTICE("RIF src MAC not provided; using switch src MAC for BVI of vlan %u", vlan_id); + } //Create BVI interface create_bvi_interface(mac_addr,vlan_id); @@ -698,10 +723,11 @@ sai_status_t SwitchVpp::vpp_create_lag( mode = VPP_BOND_API_MODE_XOR; lb = VPP_BOND_API_LB_ALGO_L34_INNER; - create_bond_interface(bond_id, mode, lb, &swif_idx); - if (swif_idx == static_cast(~0)) + int ret = create_bond_interface(bond_id, mode, lb, &swif_idx); + if (ret != 0 || swif_idx == static_cast(~0) || swif_idx == 0) { - SWSS_LOG_ERROR("failed to create bond interface in VPP for %s", sai_serialize_object_id(lag_id).c_str()); + SWSS_LOG_ERROR("failed to create bond interface in VPP for %s (ret=%d, swif_idx=%u)", + sai_serialize_object_id(lag_id).c_str(), ret, swif_idx); return SAI_STATUS_FAILURE; } diff --git a/vslib/vpp/SwitchVppHostif.cpp b/vslib/vpp/SwitchVppHostif.cpp index 8d1c52e3bc..37e3365018 100644 --- a/vslib/vpp/SwitchVppHostif.cpp +++ b/vslib/vpp/SwitchVppHostif.cpp @@ -1,6 +1,7 @@ #include "SwitchVpp.h" #include "HostInterfaceInfo.h" #include "EventPayloadNotification.h" +#include "SwitchVppUtils.h" #include "meta/sai_serialize.h" #include "meta/NotificationPortStateChange.h" @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -358,6 +360,27 @@ sai_status_t SwitchVpp::vs_create_hostif_tap_interface( configure_lcp_interface(hwif_name, dev, true); + /* + * Re-apply the port's configured admin state to the freshly created VPP host + * interface. The SAI port admin state may have been set (e.g. during port + * bring-up) before this host interface existed; at that point + * vpp_set_interface_state() could not resolve the hwif name (no tap yet) and + * the admin-up was silently dropped. Without re-applying it here the VPP + * host-interface stays admin-down, which keeps the paired kernel netdev (and + * therefore the PTF-side veth carrier) down, causing ENETDOWN on transmit. + */ + { + sai_attribute_t admin_attr; + admin_attr.id = SAI_PORT_ATTR_ADMIN_STATE; + if (get(SAI_OBJECT_TYPE_PORT, obj_id, 1, &admin_attr) == SAI_STATUS_SUCCESS && + admin_attr.value.booldata) + { + interface_set_state(hwif_name, true); + SWSS_LOG_NOTICE("Applied admin-up to VPP host interface %s for port %s", + hwif_name, sai_serialize_object_id(obj_id).c_str()); + } + } + { bool link_up = false; @@ -538,6 +561,23 @@ sai_status_t SwitchVpp::vs_remove_hostif_tap_interface( m_hostif_info_map.erase(it); } + /* + * Tear down the VPP state that vs_create_hostif_tap_interface() set up, so a + * later re-create of the same host interface in the SAME saiserver process + * starts from a clean slate. Without this the leftover linux-cp pair and + * enabled IPv6 make the next create fail (config_lcp_hostif / ip6-enable + * return VALUE_EXIST), which previously required a full backend restart per + * test to avoid. Both calls are idempotent (vpp_normalize_ret tolerates + * NO_SUCH_ENTRY on delete), so removing a partially-created hostif is safe. + */ + const char *hwif_name = tap_to_hwif_name(name.c_str()); + + if (hwif_name != nullptr) + { + sw_interface_ip6_enable_disable(hwif_name, false); + configure_lcp_interface(hwif_name, name.c_str(), false); + } + removeIfNameToPortId(name); if (port_id != SAI_NULL_OBJECT_ID) @@ -611,6 +651,30 @@ void SwitchVpp::populate_if_mapping() fclose(fp); } +static bool bond_tap_to_hwif_name( + _In_ const char *name, + _Out_ std::string &bond_hwif) +{ + SWSS_LOG_ENTER(); + + if (name[0] != 'b' || name[1] != 'e' || name[2] == '\0') + { + return false; + } + + for (const char *p = name + 2; *p != '\0'; p++) + { + if (!isdigit(static_cast(*p))) + { + return false; + } + } + + bond_hwif = std::string(BONDETHERNET_PREFIX) + (name + 2); + + return true; +} + const char* SwitchVpp::tap_to_hwif_name( _In_ const char *name) { @@ -624,6 +688,15 @@ const char* SwitchVpp::tap_to_hwif_name( if (it == m_hostif_hwif_map.end()) { + static thread_local std::string bond_hwif; + + if (bond_tap_to_hwif_name(name, bond_hwif)) + { + SWSS_LOG_DEBUG("Mapped bond tap %s to hwif %s", name, bond_hwif.c_str()); + + return bond_hwif.c_str(); + } + SWSS_LOG_ERROR("failed to find hwif info entry for hostif device: %s", tap_name.c_str()); return "Unknown"; diff --git a/vslib/vpp/SwitchVppNbr.cpp b/vslib/vpp/SwitchVppNbr.cpp index 8bf84cb911..61cb5ec6b1 100644 --- a/vslib/vpp/SwitchVppNbr.cpp +++ b/vslib/vpp/SwitchVppNbr.cpp @@ -15,10 +15,41 @@ #include #include "vppxlate/SaiVppXlate.h" +#include "SwitchVppNexthop.h" using namespace saivs; -sai_status_t SwitchVpp::addRemoveIpNbr( +void create_route_prefix_entry(sai_route_entry_t *route_entry, vpp_ip_route_t *ip_route); +void create_vpp_nexthop_entry(nexthop_grp_member_t *nxt_grp_member, const char *hwif_name, + vpp_nexthop_type_e type, vpp_ip_nexthop_t *vpp_nexthop); + +#define CHECK_STATUS_QUIET(status) { \ + sai_status_t _status = (status); \ + if (_status != SAI_STATUS_SUCCESS) { return _status; } } + +static sai_ip_prefix_t make_neighbor_host_prefix(const sai_ip_address_t &addr) +{ + SWSS_LOG_ENTER(); + + sai_ip_prefix_t prefix; + + prefix.addr_family = addr.addr_family; + prefix.addr = addr.addr; + + if (addr.addr_family == SAI_IP_ADDR_FAMILY_IPV4) + { + memset(&prefix.mask.ip4, 0xFF, sizeof(prefix.mask.ip4)); + } + else + { + memset(prefix.mask.ip6, 0xFF, sizeof(prefix.mask.ip6)); + } + + return prefix; +} + +static bool neighbor_no_host_route( + SwitchVpp *switch_db, _In_ const std::string &serializedObjectId, _In_ uint32_t attr_count, _In_ const sai_attribute_t *attr_list, @@ -26,35 +57,76 @@ sai_status_t SwitchVpp::addRemoveIpNbr( { SWSS_LOG_ENTER(); + sai_attribute_t attr; + attr.id = SAI_NEIGHBOR_ENTRY_ATTR_NO_HOST_ROUTE; + + if (is_add && attr_count > 0) + { + SaiCachedObject nbr_obj(switch_db, SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, serializedObjectId, attr_count, attr_list); + + if (nbr_obj.get_attr(attr) == SAI_STATUS_SUCCESS) + { + return attr.value.booldata; + } + } + else if (!is_add) + { + auto nbr_obj = switch_db->get_sai_object(SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, serializedObjectId); + + if (nbr_obj != nullptr && nbr_obj->get_attr(attr) == SAI_STATUS_SUCCESS) + { + return attr.value.booldata; + } + } + + return false; +} + +sai_status_t SwitchVpp::addRemoveIpNbr( + _In_ const std::string &serializedObjectId, + _In_ uint32_t attr_count, + _In_ const sai_attribute_t *attr_list, + _In_ bool is_add, + _In_ bool program_adjacency, + _In_ bool program_host_route) +{ + SWSS_LOG_ENTER(); + + if (program_adjacency == false && program_host_route == false) + { + return SAI_STATUS_SUCCESS; + } + sai_attribute_t attr; sai_neighbor_entry_t nbr_entry; sai_deserialize_neighbor_entry(serializedObjectId, nbr_entry); - attr.id = SAI_ROUTER_INTERFACE_ATTR_PORT_ID; - + attr.id = SAI_ROUTER_INTERFACE_ATTR_TYPE; CHECK_STATUS(get(SAI_OBJECT_TYPE_ROUTER_INTERFACE, nbr_entry.rif_id, 1, &attr)); - auto port_obj_type = objectTypeQuery(attr.value.oid); - if (port_obj_type != SAI_OBJECT_TYPE_PORT && port_obj_type != SAI_OBJECT_TYPE_LAG) + int32_t rif_type = attr.value.s32; + + if (rif_type != SAI_ROUTER_INTERFACE_TYPE_SUB_PORT && + rif_type != SAI_ROUTER_INTERFACE_TYPE_PORT) { + SWSS_LOG_NOTICE("Skipping neighbor VPP programming for RIF type %d", rif_type); return SAI_STATUS_SUCCESS; } - auto port_oid = attr.value.oid; - attr.id = SAI_ROUTER_INTERFACE_ATTR_TYPE; + attr.id = SAI_ROUTER_INTERFACE_ATTR_PORT_ID; CHECK_STATUS(get(SAI_OBJECT_TYPE_ROUTER_INTERFACE, nbr_entry.rif_id, 1, &attr)); - if (attr.value.s32 != SAI_ROUTER_INTERFACE_TYPE_SUB_PORT && - attr.value.s32 != SAI_ROUTER_INTERFACE_TYPE_PORT) - { - SWSS_LOG_NOTICE("Skipping neighbor add for attr type %d", attr.value.s32); + auto port_obj_type = objectTypeQuery(attr.value.oid); + if (port_obj_type != SAI_OBJECT_TYPE_PORT && port_obj_type != SAI_OBJECT_TYPE_LAG) + { return SAI_STATUS_SUCCESS; } + auto port_oid = attr.value.oid; uint16_t vlan_id = 0; - if (attr.value.s32 == SAI_ROUTER_INTERFACE_TYPE_SUB_PORT) + if (rif_type == SAI_ROUTER_INTERFACE_TYPE_SUB_PORT) { attr.id = SAI_ROUTER_INTERFACE_ATTR_OUTER_VLAN_ID; @@ -62,70 +134,137 @@ sai_status_t SwitchVpp::addRemoveIpNbr( vlan_id = attr.value.u16; } - sai_mac_t nbr_mac; - bool no_mac = true; + std::string hwif_name; + bool found = vpp_get_hwif_name(port_oid, vlan_id, hwif_name); + if (found == false) + { + SWSS_LOG_ERROR("hw interface for port/lag id %s not found", serializedObjectId.c_str()); + return SAI_STATUS_FAILURE; + } - if (is_add) + if (program_host_route) { - for (uint32_t i = 0; i < attr_count; i++) + if (neighbor_no_host_route(this, serializedObjectId, attr_count, attr_list, is_add)) { - switch (attr_list[i].id) - { - case SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS: - memcpy(nbr_mac, attr_list[i].value.mac, sizeof(sai_mac_t)); - no_mac = false; - break; + return SAI_STATUS_SUCCESS; + } - default: - break; - } + attr.id = SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID; + CHECK_STATUS_QUIET(get(SAI_OBJECT_TYPE_ROUTER_INTERFACE, nbr_entry.rif_id, 1, &attr)); + + sai_object_id_t vr_id = attr.value.oid; + + std::shared_ptr vrf = vpp_get_ip_vrf(vr_id); + uint32_t vrf_id = (vrf == nullptr) ? 0 : vrf->m_vrf_id; + + sai_route_entry_t route_entry; + route_entry.switch_id = nbr_entry.switch_id; + route_entry.vr_id = vr_id; + route_entry.destination = make_neighbor_host_prefix(nbr_entry.ip_address); + + vpp_ip_route_t *ip_route = (vpp_ip_route_t *) + calloc(1, sizeof(vpp_ip_route_t) + sizeof(vpp_ip_nexthop_t)); + + if (!ip_route) + { + return SAI_STATUS_FAILURE; } - } else { - attr.id = SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS; - if (get(SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, serializedObjectId, 1, &attr) == SAI_STATUS_SUCCESS) { - memcpy(nbr_mac, attr.value.mac, sizeof(sai_mac_t)); - no_mac = false; + create_route_prefix_entry(&route_entry, ip_route); + ip_route->vrf_id = vrf_id; + ip_route->is_multipath = false; + ip_route->nexthop_cnt = 1; + + nexthop_grp_member_t member; + memset(&member, 0, sizeof(member)); + member.addr = nbr_entry.ip_address; + member.rif_oid = nbr_entry.rif_id; + member.weight = 1; + // Force ip_route_add_del() to resolve the egress interface from hwif_name. + // sw_if_index 0 is VPP's local0: leaving it 0 (from memset) makes the host + // route's path point at local0 and the packet is dropped, even though the + // neighbor adjacency over the real interface exists. ~0 selects the + // hwif_name lookup branch (same convention as fillNHGrpMember()). + member.sw_if_index = (uint32_t) ~0; + + create_vpp_nexthop_entry(&member, hwif_name.c_str(), VPP_NEXTHOP_NORMAL, &ip_route->nexthop[0]); + + init_vpp_client(); + int ret = ip_route_add_del(ip_route, is_add); + + SWSS_LOG_NOTICE("%s neighbor host route %s status %d vrf %u hwif %s", + (is_add ? "Add" : "Remove"), serializedObjectId.c_str(), ret, vrf_id, hwif_name.c_str()); + + free(ip_route); + + if (ret != 0) + { + return SAI_STATUS_FAILURE; } } - if (no_mac == true) + if (program_adjacency) { - SWSS_LOG_ERROR("No mac address passed for neighbor %s", serializedObjectId.c_str()); - return SAI_STATUS_FAILURE; - } + sai_mac_t nbr_mac; + bool no_mac = true; - std::string hwif_name; - bool found = vpp_get_hwif_name(port_oid, vlan_id, hwif_name); - if (found == false) - { - SWSS_LOG_ERROR("hw interface for port/lag id %s not found", serializedObjectId.c_str()); - return SAI_STATUS_FAILURE; - } + if (is_add) + { + for (uint32_t i = 0; i < attr_count; i++) + { + switch (attr_list[i].id) + { + case SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS: + memcpy(nbr_mac, attr_list[i].value.mac, sizeof(sai_mac_t)); + no_mac = false; + break; + + default: + break; + } + } + } + else + { + attr.id = SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS; - const char *vpp_ifname = hwif_name.c_str(); - init_vpp_client(); + if (get(SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, serializedObjectId, 1, &attr) == SAI_STATUS_SUCCESS) + { + memcpy(nbr_mac, attr.value.mac, sizeof(sai_mac_t)); + no_mac = false; + } + } - switch (nbr_entry.ip_address.addr_family) { - case SAI_IP_ADDR_FAMILY_IPV4: - struct sockaddr_in sin; + if (no_mac == true) + { + SWSS_LOG_ERROR("No mac address passed for neighbor %s", serializedObjectId.c_str()); + return SAI_STATUS_FAILURE; + } - sin.sin_family = AF_INET; - sin.sin_addr.s_addr = nbr_entry.ip_address.addr.ip4; + const char *vpp_ifname = hwif_name.c_str(); + init_vpp_client(); - ip4_nbr_add_del(vpp_ifname, ~0, &sin, false, false, nbr_mac, is_add); + switch (nbr_entry.ip_address.addr_family) { + case SAI_IP_ADDR_FAMILY_IPV4: + struct sockaddr_in sin; - break; + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = nbr_entry.ip_address.addr.ip4; - case SAI_IP_ADDR_FAMILY_IPV6: - struct sockaddr_in6 sin6; + ip4_nbr_add_del(vpp_ifname, ~0, &sin, false, false, nbr_mac, is_add); - sin6.sin6_family = AF_INET6; - memcpy(sin6.sin6_addr.s6_addr, nbr_entry.ip_address.addr.ip6, sizeof(sin6.sin6_addr.s6_addr)); + break; - ip6_nbr_add_del(vpp_ifname, ~0, &sin6, false, false, nbr_mac, is_add); + case SAI_IP_ADDR_FAMILY_IPV6: + struct sockaddr_in6 sin6; - break; + sin6.sin6_family = AF_INET6; + memcpy(sin6.sin6_addr.s6_addr, nbr_entry.ip_address.addr.ip6, sizeof(sin6.sin6_addr.s6_addr)); + + ip6_nbr_add_del(vpp_ifname, ~0, &sin6, false, false, nbr_mac, is_add); + + break; + } } return SAI_STATUS_SUCCESS; @@ -158,11 +297,15 @@ sai_status_t SwitchVpp::addIpNbr( if (is_ip_nbr_active() == true) { SWSS_LOG_NOTICE("Add neighbor in VS %s", serializedObjectId.c_str()); - addRemoveIpNbr(serializedObjectId, attr_count, attr_list, true); + CHECK_STATUS(addRemoveIpNbr(serializedObjectId, attr_count, attr_list, true, true, false)); } CHECK_STATUS(create_internal(SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, serializedObjectId, switch_id, attr_count, attr_list)); + if (is_ip_nbr_active() == true) { + CHECK_STATUS(addRemoveIpNbr(serializedObjectId, attr_count, attr_list, true, false, true)); + } + return SAI_STATUS_SUCCESS; } @@ -173,7 +316,8 @@ sai_status_t SwitchVpp::removeIpNbr( if (is_ip_nbr_active() == true) { SWSS_LOG_NOTICE("Remove neighbor in VS %s", serializedObjectId.c_str()); - addRemoveIpNbr(serializedObjectId, 0, NULL, false); + CHECK_STATUS(addRemoveIpNbr(serializedObjectId, 0, NULL, false, false, true)); + CHECK_STATUS(addRemoveIpNbr(serializedObjectId, 0, NULL, false, true, false)); } CHECK_STATUS(remove_internal(SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, serializedObjectId)); diff --git a/vslib/vpp/SwitchVppNexthop.cpp b/vslib/vpp/SwitchVppNexthop.cpp index a8935014f7..f6aaac7d83 100644 --- a/vslib/vpp/SwitchVppNexthop.cpp +++ b/vslib/vpp/SwitchVppNexthop.cpp @@ -286,7 +286,9 @@ SwitchVpp::createNexthopGroupMember( //call create_internal to update the mapping from NHG to NHG_MBRs, which is used to update the routes status = create_internal(SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER, serializedObjectId, switch_id, attr_count, attr_list); - if (status != SAI_STATUS_SUCCESS) { + if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) { + SWSS_LOG_NOTICE("NHG member %s already exists; continuing path update", serializedObjectId.c_str()); + } else if (status != SAI_STATUS_SUCCESS) { return status; } @@ -319,8 +321,8 @@ SwitchVpp::removeNexthopGroupMember( auto nhg_mbr_obj = get_sai_object(SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER, serializedObjectId); if (!nhg_mbr_obj) { - SWSS_LOG_ERROR("Failed to find SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER SaiObject: %s", serializedObjectId.c_str()); - return SAI_STATUS_FAILURE; + SWSS_LOG_NOTICE("NHG member %s already absent; treating remove as success", serializedObjectId.c_str()); + return SAI_STATUS_SUCCESS; } attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID; CHECK_STATUS_QUIET(nhg_mbr_obj->get_mandatory_attr(attr)); @@ -344,6 +346,10 @@ SwitchVpp::removeNexthopGroupMember( //call remove_internal to update the mapping from NHG to NHG_MBRs status = remove_internal(SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER, serializedObjectId); + if (status == SAI_STATUS_ITEM_NOT_FOUND) { + SWSS_LOG_NOTICE("NHG member %s already removed from DB; treating as success", serializedObjectId.c_str()); + return SAI_STATUS_SUCCESS; + } if (status != SAI_STATUS_SUCCESS) { return status; } diff --git a/vslib/vpp/SwitchVppRif.cpp b/vslib/vpp/SwitchVppRif.cpp index 2a07d209b9..d10a8a67f1 100644 --- a/vslib/vpp/SwitchVppRif.cpp +++ b/vslib/vpp/SwitchVppRif.cpp @@ -21,6 +21,7 @@ #include "vppxlate/SaiVppXlate.h" #include +#include #include #include #include @@ -345,31 +346,21 @@ bool SwitchVpp::vpp_get_hwif_name ( { SWSS_LOG_ENTER(); - const char *hwifname = nullptr; - char hw_bondifname[32]; - - if (objectTypeQuery(object_id) == SAI_OBJECT_TYPE_LAG) { - platform_bond_info_t bond_info; - sai_status_t status = get_lag_bond_info(object_id, bond_info); - if (status != SAI_STATUS_SUCCESS) - { - return false; - } - snprintf(hw_bondifname, sizeof(hw_bondifname), "%s%u", BONDETHERNET_PREFIX, bond_info.id); - hwifname = hw_bondifname; - } else { - std::string if_name; - bool found = getTapNameFromPortId(object_id, if_name); + std::string tap_name; - if (found == false) - { - SWSS_LOG_ERROR("host interface for port id %s not found", sai_serialize_object_id(object_id).c_str()); - return false; - } - hwifname = tap_to_hwif_name(if_name.c_str()); + if (getTapNameFromPortOrLagId(object_id, tap_name) == false) + { + SWSS_LOG_ERROR("host interface for port/lag id %s not found", + sai_serialize_object_id(object_id).c_str()); + return false; } - if (!hwifname) return false; + const char *hwifname = tap_to_hwif_name(tap_name.c_str()); + + if (hwifname == NULL || strcmp(hwifname, "Unknown") == 0) + { + return false; + } char hw_subifname[64]; const char *hw_ifname; @@ -385,6 +376,50 @@ bool SwitchVpp::vpp_get_hwif_name ( return true; } +void SwitchVpp::resyncPortOperStatus() +{ + SWSS_LOG_ENTER(); + + for (auto& kvp: m_hostif_info_map) + { + const std::string& tapname = kvp.first; + auto& info = kvp.second; + + if (!info) + { + continue; + } + + const char* dev = tapname.c_str(); + const char* hwif_name = tap_to_hwif_name(dev); + + bool link_up = false; + + if (interface_get_state(hwif_name, &link_up) != 0) + { + continue; + } + + auto prev_it = m_last_oper_up.find(tapname); + bool have_prev = (prev_it != m_last_oper_up.end()); + + if (have_prev && prev_it->second == link_up) + { + continue; + } + + m_last_oper_up[tapname] = link_up; + + auto state = link_up ? SAI_PORT_OPER_STATUS_UP : SAI_PORT_OPER_STATUS_DOWN; + + send_port_oper_status_notification(info->m_portId, state, false); + + SWSS_LOG_NOTICE("resync oper-status %s(%s) %s", + hwif_name, dev, + link_up ? "UP" : "DOWN"); + } +} + void SwitchVpp::vppProcessEvents () { SWSS_LOG_ENTER(); @@ -393,10 +428,26 @@ void SwitchVpp::vppProcessEvents () vpp_event_info_t *evp; int ret; + // One-shot level oper-status resync at startup: recover the current link + // state of any interface whose state was set before we subscribed to + // interface events (edge-only delivery would otherwise miss it). + m_operResyncDue.store(true, std::memory_order_relaxed); + while(m_run_vpp_events_thread) { nanosleep(&req, NULL); ret = vpp_sync_for_events(); SWSS_LOG_NOTICE("Checking for any VS events status %d", ret); + if (ret < 0) + { + SWSS_LOG_WARN("vpp_sync_for_events failed (%d); event socket reconnect attempted", ret); + // The read failure triggers an event-socket reconnect inside + // vpp_sync_for_events. VPP only re-delivers *future* edges after + // re-subscribe, so schedule a one-shot level resync to recover any + // edge missed during the outage (e.g. reboot/config-reload) - the + // exact case resyncPortOperStatus was added for. This is event-driven + // (not a periodic poll), so it does not exhaust the VPP client clib heap. + m_operResyncDue.store(true, std::memory_order_relaxed); + } while ((evp = vpp_ev_dequeue())) { if (evp->type == VPP_INTF_LINK_STATUS) { asyncIntfStateUpdate(evp->data.intf_status.hwif_name, @@ -414,6 +465,27 @@ void SwitchVpp::vppProcessEvents () } vpp_ev_free(evp); } + // No periodic resync here. resyncPortOperStatus() issues per-interface + // SW_INTERFACE_DUMPs; doing that every cycle exhausted VPP's client clib + // heap (os_panic in clib_mem_heap_realloc_aligned). Resync is instead + // requested only at startup and after a reconnect (above), and executed + // on the command thread via serviceDeferredOperStatusResync(). + } +} + +void SwitchVpp::serviceDeferredOperStatusResync() +{ + SWSS_LOG_ENTER(); + + // Runs on the command thread (from the create/set/remove entry points). + // resyncPortOperStatus() issues VPP binary-API SW_INTERFACE_DUMPs; the event + // thread only *requests* a resync (m_operResyncDue) on startup and after an + // event-socket reconnect - never on a periodic timer - so this performs at + // most a handful of level reads over the switch lifetime, avoiding the VPP + // client clib-heap exhaustion that a continuous poll caused. + if (m_operResyncDue.exchange(false, std::memory_order_relaxed)) + { + resyncPortOperStatus(); } } @@ -680,15 +752,24 @@ sai_status_t SwitchVpp::vpp_add_del_intf_ip_addr ( return SAI_STATUS_SUCCESS; } - if (ot != SAI_OBJECT_TYPE_PORT) + std::string if_name; + + if (ot != SAI_OBJECT_TYPE_PORT && ot != SAI_OBJECT_TYPE_LAG) { - SWSS_LOG_ERROR("SAI_ROUTER_INTERFACE_ATTR_PORT_ID=%s expected to be PORT but is: %s", + SWSS_LOG_ERROR("SAI_ROUTER_INTERFACE_ATTR_PORT_ID=%s expected to be PORT or LAG but is: %s", sai_serialize_object_id(obj_id).c_str(), sai_serialize_object_type(ot).c_str()); return SAI_STATUS_FAILURE; } + if (getTapNameFromPortOrLagId(obj_id, if_name) == false) + { + SWSS_LOG_ERROR("host interface for port/lag id %s not found", + sai_serialize_object_id(obj_id).c_str()); + return SAI_STATUS_FAILURE; + } + if (rif_type != SAI_ROUTER_INTERFACE_TYPE_SUB_PORT && rif_type != SAI_ROUTER_INTERFACE_TYPE_PORT && rif_type != SAI_ROUTER_INTERFACE_TYPE_LOOPBACK) @@ -705,14 +786,6 @@ sai_status_t SwitchVpp::vpp_add_del_intf_ip_addr ( vlan_id = attr.value.u16; } - std::string if_name; - bool found = getTapNameFromPortId(obj_id, if_name); - if (found == false) - { - SWSS_LOG_ERROR("host interface for port id %s not found", sai_serialize_object_id(obj_id).c_str()); - return SAI_STATUS_FAILURE; - } - swss::IpPrefix intf_ip_prefix; char host_subifname[32]; const char *linux_ifname; @@ -798,9 +871,14 @@ sai_status_t SwitchVpp::vpp_add_del_intf_ip_addr ( } } - const char *hwifname = tap_to_hwif_name(if_name.c_str()); - char hw_subifname[32]; const char *hw_ifname; + char hw_subifname[32]; + const char *hwifname = tap_to_hwif_name(if_name.c_str()); + + if (hwifname == NULL || strcmp(hwifname, "Unknown") == 0) + { + return SAI_STATUS_FAILURE; + } if (vlan_id) { snprintf(hw_subifname, sizeof(hw_subifname), "%s.%u", hwifname, vlan_id); diff --git a/vslib/vpp/SwitchVppRoute.cpp b/vslib/vpp/SwitchVppRoute.cpp index 84970c6b74..d2eae7a271 100644 --- a/vslib/vpp/SwitchVppRoute.cpp +++ b/vslib/vpp/SwitchVppRoute.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "vppxlate/SaiVppXlate.h" @@ -33,7 +34,7 @@ using namespace saivs; SWSS_LOG_ERROR("%s: status %d", buffer, status); \ return _status; } } -static void create_route_prefix_entry ( +void create_route_prefix_entry ( sai_route_entry_t *route_entry, vpp_ip_route_t *ip_route) { @@ -102,6 +103,45 @@ void create_vpp_nexthop_entry ( vpp_nexthop->preference = 0; } +const char* SwitchVpp::resolveNexthopMemberHwif( + _In_ const nexthop_grp_member_t *member, + _Out_ std::string &member_hwif) +{ + SWSS_LOG_ENTER(); + + member_hwif.clear(); + + if (member == NULL || member->rif_oid == SAI_NULL_OBJECT_ID) + { + return NULL; + } + + sai_attribute_t rif_attr; + uint16_t vlan_id = 0; + + rif_attr.id = SAI_ROUTER_INTERFACE_ATTR_TYPE; + if (get(SAI_OBJECT_TYPE_ROUTER_INTERFACE, member->rif_oid, 1, &rif_attr) == SAI_STATUS_SUCCESS && + rif_attr.value.s32 == SAI_ROUTER_INTERFACE_TYPE_SUB_PORT) + { + sai_attribute_t vlan_attr; + vlan_attr.id = SAI_ROUTER_INTERFACE_ATTR_OUTER_VLAN_ID; + + if (get(SAI_OBJECT_TYPE_ROUTER_INTERFACE, member->rif_oid, 1, &vlan_attr) == SAI_STATUS_SUCCESS) + { + vlan_id = vlan_attr.value.u16; + } + } + + rif_attr.id = SAI_ROUTER_INTERFACE_ATTR_PORT_ID; + if (get(SAI_OBJECT_TYPE_ROUTER_INTERFACE, member->rif_oid, 1, &rif_attr) == SAI_STATUS_SUCCESS && + vpp_get_hwif_name(rif_attr.value.oid, vlan_id, member_hwif)) + { + return member_hwif.c_str(); + } + + return NULL; +} + sai_status_t SwitchVpp::IpRouteAddRemove( _In_ const SaiObject* route_obj, _In_ bool is_add, @@ -122,7 +162,35 @@ sai_status_t SwitchVpp::IpRouteAddRemove( packet_action = attr.value.s32; } - // We should program drop routes + sai_route_entry_t route_entry; + sai_deserialize_route_entry(serializedObjectId, route_entry); + + if (packet_action == SAI_PACKET_ACTION_DROP) { + std::shared_ptr vrf = vpp_get_ip_vrf(route_entry.vr_id); + uint32_t vrf_id = vrf == nullptr ? 0 : vrf->m_vrf_id; + vpp_ip_route_t *ip_route = (vpp_ip_route_t *) + calloc(1, sizeof(vpp_ip_route_t) + sizeof(vpp_ip_nexthop_t)); + if (!ip_route) { + return SAI_STATUS_FAILURE; + } + + create_route_prefix_entry(&route_entry, ip_route); + ip_route->vrf_id = vrf_id; + ip_route->nexthop_cnt = 1; + ip_route->nexthop[0].addr.sa_family = ip_route->prefix_addr.sa_family; + ip_route->nexthop[0].sw_if_index = (uint32_t)~0; + ip_route->nexthop[0].weight = 1; + ip_route->nexthop[0].type = VPP_NEXTHOP_DROP; + + ret = ip_route_add_del_get_stats(ip_route, is_add, is_add ? stats_index : NULL); + free(ip_route); + + SWSS_LOG_NOTICE("%s drop route in VS %s status %d table %u", + is_add ? "Add" : "Remove", + serializedObjectId.c_str(), ret, vrf_id); + return ret == 0 ? SAI_STATUS_SUCCESS : SAI_STATUS_FAILURE; + } + if (packet_action != SAI_PACKET_ACTION_FORWARD) { SWSS_LOG_NOTICE("Ignoring ip route %s: action is not forward: %d", serializedObjectId.c_str(), packet_action); @@ -138,13 +206,10 @@ sai_status_t SwitchVpp::IpRouteAddRemove( } next_hop_oid = attr.value.oid; - sai_route_entry_t route_entry; const char *hwif_name = NULL; vpp_nexthop_type_e nexthop_type = VPP_NEXTHOP_NORMAL; bool config_ip_route = false; - sai_deserialize_route_entry(serializedObjectId, route_entry); - nexthop_grp_config_t *nxthop_group = NULL; if (SAI_OBJECT_TYPE_ROUTER_INTERFACE == RealObjectIdManager::objectTypeQuery(next_hop_oid)) @@ -199,9 +264,25 @@ sai_status_t SwitchVpp::IpRouteAddRemove( nxt_grp_member = nxthop_group->grp_members; + std::vector member_hwif_storage; + member_hwif_storage.reserve(nxthop_group->nmembers); + size_t i; for (i = 0; i < nxthop_group->nmembers; i++) { - create_vpp_nexthop_entry(nxt_grp_member, hwif_name, nexthop_type, &ip_route->nexthop[i]); + const char *member_hwif = hwif_name; + std::string resolved_hwif; + + if (member_hwif == NULL) + { + member_hwif = resolveNexthopMemberHwif(nxt_grp_member, resolved_hwif); + if (member_hwif != NULL) + { + member_hwif_storage.push_back(std::move(resolved_hwif)); + member_hwif = member_hwif_storage.back().c_str(); + } + } + + create_vpp_nexthop_entry(nxt_grp_member, member_hwif, nexthop_type, &ip_route->nexthop[i]); nxt_grp_member++; } ip_route->nexthop_cnt = nxthop_group->nmembers; @@ -391,7 +472,10 @@ sai_status_t SwitchVpp::IpRoutePathAddRemove( ip_route->is_multipath = true; // Tell VPP to add/remove a path, not replace the route ip_route->nexthop_cnt = 1; - create_vpp_nexthop_entry(member, NULL, VPP_NEXTHOP_NORMAL, &ip_route->nexthop[0]); + std::string member_hwif_str; + const char *member_hwif = resolveNexthopMemberHwif(member, member_hwif_str); + + create_vpp_nexthop_entry(member, member_hwif, VPP_NEXTHOP_NORMAL, &ip_route->nexthop[0]); int ret = ip_route_add_del_get_stats(ip_route, is_add, stats_index); diff --git a/vslib/vpp/vppxlate/SaiVppXlate.c b/vslib/vpp/vppxlate/SaiVppXlate.c index 3cd77edf28..cfc779696c 100644 --- a/vslib/vpp/vppxlate/SaiVppXlate.c +++ b/vslib/vpp/vppxlate/SaiVppXlate.c @@ -376,10 +376,11 @@ void os_exit(int code) {} #include "../SaiVppLog.h" /* - * Normalize VPP return code for idempotent operations. - * If the operation is a delete and VPP returns NO_SUCH_ENTRY, the entry is - * already gone; if it is an add and VPP returns VALUE_EXIST, the entry is - * already present. In both cases treat it as success. + * Normalize VPP return code so add/delete operations are idempotent. + * - delete + NO_SUCH_ENTRY: the entry is already gone — treat as success. + * - add + VALUE_EXIST: the entry already exists — treat as success. + * This lets the SAI backend re-run a create/remove cycle in one process + * (e.g. host-interface recreate) without failing on leftover VPP state. */ static inline int vpp_normalize_ret(int ret, bool is_del, const char *func) { @@ -390,6 +391,10 @@ static inline int vpp_normalize_ret(int ret, bool is_del, const char *func) SAIVPP_INFO("%s: ignoring VALUE_EXIST(%d) on add", func, ret); ret = 0; } + if (!is_del && ret == VNET_API_ERROR_VALUE_EXIST) { + SAIVPP_INFO("%s: ignoring VALUE_EXIST(%d) on add", func, ret); + ret = 0; + } return ret; } @@ -414,12 +419,24 @@ do { \ */ #define WR(ret) \ do { \ - f64 timeout = vat_time_now (vam) + 1.0; \ + f64 start_time = vat_time_now (vam); \ + f64 hard_deadline = start_time + 10.0; \ + f64 idle_deadline = start_time + 1.0; \ socket_client_main_t *scm = vam->socket_client_main; \ + int _wr_rv; \ ret = -99; \ - while (vat_time_now (vam) < timeout) { \ - if (scm && scm->socket_enable) \ - vl_socket_client_read (5); \ + while (1) { \ + f64 now = vat_time_now (vam); \ + if (now >= hard_deadline || now >= idle_deadline) \ + break; \ + if (scm && scm->socket_enable) { \ + _wr_rv = vl_socket_client_read (1); \ + if (_wr_rv == 0) { \ + idle_deadline = vat_time_now (vam) + 1.0; \ + if (idle_deadline > hard_deadline) \ + idle_deadline = hard_deadline; \ + } \ + } \ if (vam->result_ready == 1) { \ ret = vam->retval; \ break; \ @@ -428,6 +445,83 @@ do { \ } \ } while(0); +/* + * Event-connection variants of M / S / PING / WR. + * + * The asynchronous VPP event stream is carried on a dedicated binary-API + * socket (event_socket_client_main / vat_event_main) that is independent of + * the synchronous request/reply socket used by the command path. These macros + * therefore operate on the explicit "scm" passed via vam->socket_client_main + * using the vl_socket_client_*2() APIs instead of the global-socket helpers. + */ +#define M_EV(T, mp) \ +do { \ + socket_client_main_t *scm = vam->socket_client_main; \ + vam->result_ready = 0; \ + if (scm && scm->socket_enable) \ + mp = vl_socket_client_msg_alloc2 (scm, (int)sizeof(*mp)); \ + else \ + mp = vl_msg_api_alloc_as_if_client((int)sizeof(*mp)); \ + clib_memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T+__plugin_msg_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define PING_EV(mp_ping) \ +do { \ + socket_client_main_t *scm = vam->socket_client_main; \ + if (scm && scm->socket_enable) \ + mp_ping = vl_socket_client_msg_alloc2 (scm, (int)sizeof (*mp_ping)); \ + else \ + mp_ping = vl_msg_api_alloc_as_if_client ((int)sizeof (*mp_ping)); \ + clib_memset (mp_ping, 0, sizeof (*mp_ping)); \ + mp_ping->_vl_msg_id = htons (VL_API_CONTROL_PING + 1); \ + mp_ping->client_index = vam->my_client_index; \ + vam->result_ready = 0; \ + if (scm) \ + scm->control_pings_outstanding++; \ +} while(0); + +#define S_EV(mp) \ +do { \ + socket_client_main_t *scm = vam->socket_client_main; \ + if (scm && scm->socket_enable) \ + vl_socket_client_write2 (scm); \ + else \ + vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp); \ +} while (0); + +#define WR_EV(ret) \ +do { \ + f64 start_time = vat_time_now (vam); \ + f64 hard_deadline = start_time + 10.0; \ + f64 idle_deadline = start_time + 1.0; \ + socket_client_main_t *scm = vam->socket_client_main; \ + ret = -99; \ + while (1) { \ + f64 now = vat_time_now (vam); \ + if (now >= hard_deadline || now >= idle_deadline) \ + break; \ + if (scm && scm->socket_enable) { \ + int _wr_rv = vl_socket_client_read2 (scm, 1); \ + if (_wr_rv < 0) { \ + ret = _wr_rv; \ + break; \ + } \ + if (_wr_rv == 0) { \ + idle_deadline = vat_time_now (vam) + 1.0; \ + if (idle_deadline > hard_deadline) \ + idle_deadline = hard_deadline; \ + } \ + } \ + if (vam->result_ready == 1) { \ + ret = vam->retval; \ + break; \ + } \ + vat_suspend (vam->vlib_main, 1e-5); \ + } \ +} while(0); + #define VPP_MAX_CTX 16 #define VPP_CTX_INDEX_MASK 0xff #define VPP_CTX_GENERATION_SHIFT 8 @@ -482,6 +576,9 @@ static void vpp_evq_init () static int vpp_acl_counters_enable_disable(bool enable); static int vpp_intf_events_enable_disable(bool enable); static int vpp_bfd_events_enable_disable(bool enable); +static int vpp_event_client_setup(void); +static int vpp_event_connect(void); +static int vpp_event_reconnect(void); static int vpp_bfd_udp_enable_multihop(); static int vpp_lcp_ethertype_enable(u16 ethertype); @@ -489,7 +586,19 @@ static pthread_mutex_t vpp_mutex; void vpp_mutex_lock_init () { - pthread_mutex_init(&vpp_mutex, NULL); + /* + * Recursive so the command path can nest EVENT_LOCK inside VPP_LOCK (e.g. + * event registration/reconnect issued while holding VPP_LOCK). EVENT_LOCK + * now maps onto this same mutex (see EVENT_LOCK below): the command path and + * the background event thread both allocate on VPP's process-global, + * non-thread-safe clib heap, so they must be mutually exclusive to avoid + * heap corruption (os_panic in clib_mem_heap_realloc_aligned). + */ + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&vpp_mutex, &attr); + pthread_mutexattr_destroy(&attr); } void vpp_mutex_lock () @@ -505,6 +614,65 @@ void vpp_mutex_unlock () #define VPP_LOCK() vpp_mutex_lock() #define VPP_UNLOCK() vpp_mutex_unlock() +/* + * Legacy dedicated event-connection mutex. No longer used for locking: EVENT_LOCK + * now maps onto the recursive vpp_mutex (see EVENT_LOCK below) so that command-path + * and event-thread clib allocations are mutually exclusive on VPP's shared, + * non-thread-safe clib heap. Retained only to keep the init/teardown surface + * unchanged; kept unused otherwise. + */ +static pthread_mutex_t vpp_event_mutex; + +void vpp_event_mutex_lock_init () +{ + pthread_mutex_init(&vpp_event_mutex, NULL); +} + +void vpp_event_mutex_lock () +{ + pthread_mutex_lock(&vpp_event_mutex); +} + +void vpp_event_mutex_unlock () +{ + pthread_mutex_unlock(&vpp_event_mutex); +} + +/* + * EVENT_LOCK maps onto the same recursive vpp_mutex as VPP_LOCK. The event + * connection (vat_event_main) and the command connection (vat_main) both + * allocate on VPP's process-global, non-thread-safe clib heap; a background + * event-thread clib allocation racing a command-thread clib allocation corrupts + * the heap and crashes (os_panic in clib_mem_heap_realloc_aligned). Serializing + * every VPP client/clib operation under one recursive mutex prevents that. The + * mutex is recursive so the command path may still nest EVENT_LOCK inside + * VPP_LOCK (event registration/reconnect). No command operation blocks waiting + * on an unsolicited event, so this single lock cannot deadlock. + */ +#define EVENT_LOCK() vpp_mutex_lock() +#define EVENT_UNLOCK() vpp_mutex_unlock() + +/* + * Leaf lock protecting the shared interface lookup tables + * (vat_main.sw_if_index_by_interface_name, interface_name_by_sw_index and + * link_speed_by_sw_index). These tables are read and written from both the + * synchronous command path (under VPP_LOCK) and the asynchronous event + * handler (under EVENT_LOCK), so neither connection lock on its own provides + * mutual exclusion over them; a concurrent read during a write/rehash/free + * corrupts the hash and crashes syncd. + * + * Lock ordering: this is a leaf lock. It may be acquired while holding + * VPP_LOCK or EVENT_LOCK, but code holding it must never acquire VPP_LOCK, + * EVENT_LOCK, or re-acquire this lock (keep the critical sections to the raw + * hash operations only). This keeps the EVENT_LOCK-never-acquires-VPP_LOCK + * ordering rule above intact. Statically initialized so it is valid before + * any thread (including the background event thread) starts. + */ +static pthread_mutex_t vpp_intf_table_mutex = PTHREAD_MUTEX_INITIALIZER; + +#define INTF_TABLE_LOCK() pthread_mutex_lock(&vpp_intf_table_mutex) +#define INTF_TABLE_UNLOCK() pthread_mutex_unlock(&vpp_intf_table_mutex) + /* * Right now configuration is done synchronously in a single thread. * When the need arises for multiple requests in pipeline we can move to a pool to @@ -570,6 +738,32 @@ vat_main_t vat_main; uword *interface_name_by_sw_index = NULL; uword *link_speed_by_sw_index = NULL; +/* + * Dedicated binary-API connection for the asynchronous VPP event stream + * (sw_interface_event / bfd events). Owned by the background event-polling + * thread and isolated from the synchronous request/reply connection so the + * unsolicited event flood cannot starve or deadlock the command path. + */ +static socket_client_main_t event_socket_client_main; +vat_main_t vat_event_main; + +static int event_client_connected = 0; +static int event_mutex_initialized = 0; + +/* + * Per-thread "current" vat_main. Reply handlers run in the context of whatever + * thread is reading its connection's socket; they use cur_vam() so that a reply + * read on a given connection updates only that connection's result state. + * Defaults to the command connection (&vat_main) when unset. + */ +static __thread vat_main_t *tl_cur_vam; + +static inline vat_main_t *cur_vam (void) +{ + return tl_cur_vam ? tl_cur_vam : &vat_main; +} + + f64 vat_time_now (vat_main_t * vam) { @@ -691,7 +885,7 @@ vl_noop_handler (void *mp) static void set_reply_status (int retval) { - vat_main_t *vam = &vat_main; + vat_main_t *vam = cur_vam(); if (vam->async_mode) { @@ -706,14 +900,14 @@ static void set_reply_status (int retval) static void set_reply_sw_if_index (vl_api_interface_index_t sw_if_index) { - vat_main_t *vam = &vat_main; + vat_main_t *vam = cur_vam(); vam->sw_if_index = sw_if_index; } static void vl_api_control_ping_reply_t_handler (vl_api_control_ping_reply_t *mp) { - vat_main_t *vam = &vat_main; + vat_main_t *vam = cur_vam(); set_reply_status((int)ntohl ((uint32_t)mp->retval)); @@ -734,15 +928,25 @@ vl_api_sw_interface_event_t_handler (vl_api_sw_interface_event_t *mp) uint32_t flags, sw_if_index; uword *ptr; - sw_if_index = htonl(mp->sw_if_index); + sw_if_index = ntohl(mp->sw_if_index); + + /* + * Copy the interface name out while holding the table lock; the pointer + * returned by hash_get points into interface_name_by_sw_index, which the + * command path may concurrently rewrite/free under VPP_LOCK. + */ + char hw_ifname[64]; + INTF_TABLE_LOCK(); ptr = hash_get(interface_name_by_sw_index, sw_if_index); if (NULL == ptr) { + INTF_TABLE_UNLOCK(); SAIVPP_INFO("vpp cannot get interface name for sw index %u", sw_if_index); return; } - const char *hw_ifname = (const char *) ptr[0]; + snprintf(hw_ifname, sizeof(hw_ifname), "%s", (const char *) ptr[0]); + INTF_TABLE_UNLOCK(); - flags = htonl(mp->flags); + flags = ntohl(mp->flags); if (flags & IF_STATUS_API_FLAG_ADMIN_UP && !(flags & IF_STATUS_API_FLAG_LINK_UP)) { return; @@ -764,7 +968,7 @@ vl_api_sw_interface_event_t_handler (vl_api_sw_interface_event_t *mp) vpp_intf_status_t *stp = &evinfo->data.intf_status; stp->link_up = link_up; - strncpy(stp->hwif_name, hw_ifname, sizeof(stp->hwif_name) -1); + snprintf(stp->hwif_name, sizeof(stp->hwif_name), "%s", hw_ifname); vpp_ev_enqueue(evinfo); } @@ -785,12 +989,14 @@ vl_api_sw_interface_details_t_handler (vl_api_sw_interface_details_t *mp) vat_main_t *vam = &vat_main; u8 *s = format (0, "%s%c", mp->interface_name, 0); + INTF_TABLE_LOCK(); hash_set_mem (vam->sw_if_index_by_interface_name, s, ntohl (mp->sw_if_index)); hash_set (interface_name_by_sw_index, ntohl (mp->sw_if_index), s); /* Save link speed (in Kbps) per interface */ hash_set (link_speed_by_sw_index, ntohl (mp->sw_if_index), ntohl (mp->link_speed)); + INTF_TABLE_UNLOCK(); /* In sub interface case, fill the sub interface table entry */ if (mp->sw_if_index != mp->sup_sw_if_index) @@ -1371,7 +1577,14 @@ vl_api_sr_set_encap_source_reply_t_handler(vl_api_sr_set_encap_source_reply_t *m _(MEMCLNT_MSG_ID(GET_FIRST_MSG_ID_REPLY), get_first_msg_id_reply) \ _(MEMCLNT_MSG_ID(CONTROL_PING_REPLY), control_ping_reply) -static u16 interface_msg_id_base, memclnt_msg_id_base, __plugin_msg_base; +static u16 interface_msg_id_base, memclnt_msg_id_base; +/* + * __plugin_msg_base selects the message-id base for the message currently + * being constructed. The command path and the dedicated event-connection path + * run on different threads under different locks, so this must be thread-local + * to avoid a data race between "set base" and the immediately following M()/PING(). + */ +static __thread u16 __plugin_msg_base; static u16 l2_msg_id_base, vxlan_msg_id_base, ipip_msg_id_base; static u16 tunterm_msg_id_base; static u16 bfd_msg_id_base; @@ -1673,6 +1886,105 @@ vsc_socket_connect (vat_main_t * vam, char *client_name) return 0; } +/* + * Establish the dedicated event connection. Uses the explicit-scm + * vl_socket_client_connect2() so it is a fully independent client connection + * (its own VPP client_index and socket fd). Deliberately does NOT touch the + * global api_main->my_client_index, which belongs to the command connection; + * messages on this connection carry vat_event_main.my_client_index via the + * M_EV/PING_EV macros. + */ +static int +vsc_event_socket_connect (vat_main_t * vam, char *client_name) +{ + int rv; + vam->socket_client_main = &event_socket_client_main; + if ((rv = vl_socket_client_connect2 (&event_socket_client_main, + (char *) vam->socket_name, + client_name, + 0 /* default socket rx, tx buffer */ ))) + return rv; + + /* vpp expects the client index in network order */ + vam->my_client_index = (u32)htonl (event_socket_client_main.client_index); + return 0; +} + +static int init_vpp_event_client (void) +{ + return vpp_event_connect(); +} + +static int +vpp_event_client_setup (void) +{ + vat_main_t *vam = &vat_event_main; + static int vam_setup_done = 0; + + if (!event_mutex_initialized) + { + vpp_event_mutex_lock_init(); + event_mutex_initialized = 1; + } + + if (!vam_setup_done) + { + clib_time_init (&vam->clib_time); + vam->socket_name = format (0, "%s%c", API_SOCKET_FILE, 0); + vam->vlib_main = vat_main.vlib_main; + vam_setup_done = 1; + } + + return 0; +} + +static int +vpp_event_connect (void) +{ + char client_name[] = "sonic_vpp_event_client"; + vat_main_t *vam = &vat_event_main; + + vpp_event_client_setup(); + + if (vsc_event_socket_connect(vam, client_name) == 0) + { + event_client_connected = 1; + SAIVPP_INFO("vpp event socket connect successful\n"); + return 0; + } + + event_client_connected = 0; + SAIVPP_ERROR("vpp event socket connect failed\n"); + return -1; +} + +static int +vpp_event_reconnect (void) +{ + vat_main_t *vam = &vat_event_main; + + SAIVPP_INFO("attempting vpp event socket reconnect\n"); + + EVENT_LOCK(); + vl_socket_client_disconnect2 (&event_socket_client_main); + event_client_connected = 0; + vam->result_ready = 0; + EVENT_UNLOCK(); + + if (vpp_event_connect() != 0) + return -1; + + if (vpp_intf_events_enable_disable(true) != 0) + return -1; + + if (vpp_bfd_events_enable_disable(true) != 0) + return -1; + + SAIVPP_INFO("vpp event socket reconnect successful\n"); + return 0; +} + + typedef struct { u8 *name; @@ -1691,6 +2003,14 @@ api_sw_interface_dump (vat_main_t *vam) VPP_LOCK(); + /* + * Toss and recreate the name tables atomically w.r.t. the event handler, + * which reads interface_name_by_sw_index under INTF_TABLE_LOCK. The lock is + * released before the dump below because WR() dispatches the + * sw_interface_details handler, which re-acquires INTF_TABLE_LOCK. + */ + INTF_TABLE_LOCK(); + /* Toss the old name table */ hash_foreach_pair (p, vam->sw_if_index_by_interface_name, ({ vec_add2 (nses, ns, 1); @@ -1717,6 +2037,9 @@ api_sw_interface_dump (vat_main_t *vam) /* recreate the interface name hash table */ vam->sw_if_index_by_interface_name = hash_create_string (0, sizeof (uword)); + + INTF_TABLE_UNLOCK(); + __plugin_msg_base = interface_msg_id_base; /* * Ask for all interface names. Otherwise, the epic catalog of @@ -1761,11 +2084,13 @@ dump_interface_table (vat_main_t *vam) return -99; } + INTF_TABLE_LOCK(); hash_foreach_pair (p, vam->sw_if_index_by_interface_name, ({ vec_add2 (nses, ns, 1); ns->name = (u8 *) (p->key); ns->value = (u32) p->value[0]; })); + INTF_TABLE_UNLOCK(); vec_sort_with_function (nses, name_sort_cmp); @@ -1783,13 +2108,16 @@ static u32 get_swif_idx (vat_main_t *vam, const char *ifname) hash_pair_t *p; u8 *name; u32 value; + u32 found = (u32) -1; + INTF_TABLE_LOCK(); hash_foreach_pair (p, vam->sw_if_index_by_interface_name, ({ name = (u8 *) (p->key); value = (u32) p->value[0]; - if (strcmp((char *) name, ifname) == 0) return value; + if (strcmp((char *) name, ifname) == 0) { found = value; } })); - return ((u32) -1); + INTF_TABLE_UNLOCK(); + return found; } static const char * get_swif_name (vat_main_t *vam, const u32 swif_idx) @@ -1797,13 +2125,16 @@ static const char * get_swif_name (vat_main_t *vam, const u32 swif_idx) hash_pair_t *p; u8 *name; u32 value; + const char *found = NULL; + INTF_TABLE_LOCK(); hash_foreach_pair (p, vam->sw_if_index_by_interface_name, ({ name = (u8 *) (p->key); value = (u32) p->value[0]; - if (value == swif_idx) return (const char * )name; + if (value == swif_idx) { found = (const char *) name; } })); - return NULL; + INTF_TABLE_UNLOCK(); + return found; } static int config_lcp_hostif (vat_main_t *vam, @@ -1972,9 +2303,11 @@ int init_vpp_client() vpp_base_vpe_init(); vam->socket_name = format (0, "%s%c", API_SOCKET_FILE, 0); + INTF_TABLE_LOCK(); vam->sw_if_index_by_interface_name = hash_create_string (0, sizeof (uword)); interface_name_by_sw_index = hash_create (0, sizeof (uword)); link_speed_by_sw_index = hash_create (0, sizeof (uword)); + INTF_TABLE_UNLOCK(); if (vsc_socket_connect(vam, client_name) == 0) { int rc; @@ -2006,14 +2339,24 @@ int init_vpp_client() vpp_lcp_ethertype_enable(0x0806); /* - * SONiC periodically polls the port status so currently there is no need for - * async notification. This also simplifies the synchronous design of saivpp. - * Revisit the async mechanism if there is greater reason. + * Bring up the dedicated event connection before subscribing to any + * VPP event source. The want_*_events registrations below are issued + * on this connection, so VPP delivers all unsolicited notifications to + * the event socket, keeping the command socket reply-only. */ - vpp_intf_events_enable_disable(true); - - /* Register with VPP for BFD notifications */ - vpp_bfd_events_enable_disable(true); + if (init_vpp_event_client() != 0) { + SAIVPP_ERROR("vpp event client init failed; async events disabled\n"); + } else { + /* + * SONiC periodically polls the port status so currently there is no need for + * async notification. This also simplifies the synchronous design of saivpp. + * Revisit the async mechanism if there is greater reason. + */ + vpp_intf_events_enable_disable(true); + + /* Register with VPP for BFD notifications */ + vpp_bfd_events_enable_disable(true); + } /* Enable BFD multihop support in VPP */ vpp_bfd_udp_enable_multihop(); @@ -2131,25 +2474,27 @@ int set_interface_vrf (const char *hwif_name, u32 sub_id, u32 vrf_id, bool is_ip static int vpp_intf_events_enable_disable (bool enable) { - vat_main_t *vam = &vat_main; + vat_main_t *vam = &vat_event_main; vl_api_want_interface_events_t *mp; int ret; - VPP_LOCK(); + EVENT_LOCK(); + tl_cur_vam = &vat_event_main; __plugin_msg_base = interface_msg_id_base; - M (WANT_INTERFACE_EVENTS, mp); + M_EV (WANT_INTERFACE_EVENTS, mp); mp->enable_disable = enable; mp->pid = htonl((uint32_t)getpid()); - S (mp); - WR (ret); + S_EV (mp); + WR_EV (ret); if (ret) { SAIVPP_ERROR("%s failed(%d) enable %d", __func__, ret, enable); } else { SAIVPP_INFO("%s enable %d", __func__, enable); } - VPP_UNLOCK(); + tl_cur_vam = NULL; + EVENT_UNLOCK(); return ret; } @@ -2364,6 +2709,8 @@ int ip_route_add_del_get_stats (vpp_ip_route_t *prefix, bool is_add, uint32_t *s fib_path->type = htonl(FIB_API_PATH_TYPE_NORMAL); } else if (nexthop->type == VPP_NEXTHOP_LOCAL) { fib_path->type = htonl(FIB_API_PATH_TYPE_LOCAL); + } else if (nexthop->type == VPP_NEXTHOP_DROP) { + fib_path->type = htonl(FIB_API_PATH_TYPE_DROP); } fib_path->table_id = 0; fib_path->rpf_id = htonl((uint32_t)~0); @@ -3142,13 +3489,16 @@ int vpp_get_interface_speed (const char *hwif_name, uint32_t *speed) return -EINVAL; } + INTF_TABLE_LOCK(); p = hash_get(link_speed_by_sw_index, idx); if (!p) { + INTF_TABLE_UNLOCK(); VPP_UNLOCK(); return -ENOENT; } *speed = (uint32_t) p[0]; + INTF_TABLE_UNLOCK(); if (*speed == 0 || *speed == UINT32_MAX) { VPP_UNLOCK(); @@ -3160,23 +3510,43 @@ int vpp_get_interface_speed (const char *hwif_name, uint32_t *speed) return 0; } +/* + * Synchronize with VPP via the dedicated event socket. Runs under EVENT_LOCK; + * must not acquire VPP_LOCK (see EVENT_LOCK comment above). + */ int vpp_sync_for_events () { - vat_main_t *vam = &vat_main; + vat_main_t *vam = &vat_event_main; vl_api_control_ping_t *mp_ping; int ret; - VPP_LOCK(); + if (!event_client_connected) + { + if (vpp_event_reconnect() != 0) + return -1; + } + + EVENT_LOCK(); + tl_cur_vam = &vat_event_main; + + vam->result_ready = 0; /* Use a control ping for synchronization */ __plugin_msg_base = memclnt_msg_id_base; - PING (NULL, mp_ping); - S (mp_ping); + PING_EV (mp_ping); + S_EV (mp_ping); - WR (ret); + WR_EV (ret); - VPP_UNLOCK(); + tl_cur_vam = NULL; + EVENT_UNLOCK(); + + if (ret < 0) + { + SAIVPP_ERROR("vpp_sync_for_events failed (%d), reconnecting event socket\n", ret); + vpp_event_reconnect(); + } return ret; } @@ -3340,6 +3710,12 @@ int sw_interface_ip6_enable_disable(const char *hwif_name, bool enable) WR (ret); + /* enable is an "add"; disable is a "delete". Tolerate VALUE_EXIST when + * enabling an interface whose IPv6 is already enabled (and NO_SUCH_ENTRY + * when disabling one that is already disabled) so host-interface recreate + * in a single process is idempotent. */ + ret = vpp_normalize_ret(ret, !enable, __func__); + if (ret) { SAIVPP_ERROR("%s failed(%d) %s enable %d", __func__, ret, hwif_name, enable); } else { SAIVPP_INFO("%s %s enable %d", __func__, hwif_name, enable); } @@ -4033,25 +4409,27 @@ int bfd_udp_del(bool multihop, const char *hwif_name, vpp_ip_addr_t *local_addr, static int vpp_bfd_events_enable_disable (bool enable) { - vat_main_t *vam = &vat_main; + vat_main_t *vam = &vat_event_main; vl_api_want_bfd_events_t *mp; int ret; - VPP_LOCK(); + EVENT_LOCK(); + tl_cur_vam = &vat_event_main; __plugin_msg_base = bfd_msg_id_base; - M (WANT_BFD_EVENTS, mp); + M_EV (WANT_BFD_EVENTS, mp); mp->enable_disable = enable; mp->pid = htonl((uint32_t)getpid()); - S (mp); - WR (ret); + S_EV (mp); + WR_EV (ret); if (ret) { SAIVPP_ERROR("%s failed(%d) enable %d", __func__, ret, enable); } else { SAIVPP_INFO("%s enable %d", __func__, enable); } - VPP_UNLOCK(); + tl_cur_vam = NULL; + EVENT_UNLOCK(); return ret; } @@ -4737,9 +5115,11 @@ int vpp_sw_interface_find_by_ip(vpp_ip_addr_t *search_ip, uint32_t vrf_id, // Iterates all known sw intfs to collect addresses. u32 *sw_if_idxs = NULL; hash_pair_t *p; + INTF_TABLE_LOCK(); hash_foreach_pair(p, interface_name_by_sw_index, ({ vec_add1(sw_if_idxs, (u32) p->key); })); + INTF_TABLE_UNLOCK(); for (unsigned int i = 0; i < vec_len(sw_if_idxs); i++) { /* Pre-filter: skip interfaces not in the target VRF */ diff --git a/vslib/vpp/vppxlate/SaiVppXlate.h b/vslib/vpp/vppxlate/SaiVppXlate.h index 352bfc1534..249ec7e18d 100644 --- a/vslib/vpp/vppxlate/SaiVppXlate.h +++ b/vslib/vpp/vppxlate/SaiVppXlate.h @@ -24,7 +24,8 @@ extern "C" { typedef enum { VPP_NEXTHOP_NORMAL = 1, - VPP_NEXTHOP_LOCAL = 2 + VPP_NEXTHOP_LOCAL = 2, + VPP_NEXTHOP_DROP = 3 } vpp_nexthop_type_e; typedef struct vpp_ip_addr_ {