diff --git a/.azure-pipelines/docker-sai-test-vpp/.gitignore b/.azure-pipelines/docker-sai-test-vpp/.gitignore new file mode 100644 index 0000000000..2adbd7811a --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/.gitignore @@ -0,0 +1,7 @@ +# Local-only working notes and test artifacts (not published to remote). +# devdocs/ — progress/debug logs for agent/developer context across branches +# demodocs/ — stakeholder demo notes +# results/ — JUnit XML, run logs, compatibility matrices (update README pass list when changed) +devdocs/ +demodocs/ +results/ diff --git a/.azure-pipelines/docker-sai-test-vpp/Dockerfile b/.azure-pipelines/docker-sai-test-vpp/Dockerfile new file mode 100644 index 0000000000..4b87ec8604 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/Dockerfile @@ -0,0 +1,138 @@ +FROM debian:trixie + +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_BREAK_SYSTEM_PACKAGES=1 \ + PYTHONUNBUFFERED=1 \ + SAI_PROFILE=/etc/sai/sai.profile \ + SAISERVER_PORTMAP=/etc/sai/port-map.ini \ + PTF_PORTMAP=/etc/sai/ptf-port-map.ini \ + SAI_TEST_DIR=/sai_test \ + TEST_RESULTS_DIR=/test-results + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Build from the sonic-sairedis repository root: +# docker build -f .azure-pipelines/docker-sai-test-vpp/Dockerfile -t docker-sai-test-vpp . +COPY .azure-pipelines/docker-sai-test-vpp /opt/docker-sai-test-vpp +COPY SAI/test/ptf /opt/ptf +COPY SAI/test/sai_test /sai_test + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + iproute2 \ + libboost-serialization1.83.0 \ + libpcap0.8 \ + libthrift-0.19.0t64 \ + libzmq5 \ + procps \ + python3 \ + python3-pip \ + python3-setuptools \ + python3-thrift \ + python3-wheel \ + redis-server \ + redis-tools \ + tcpdump && \ + rm -rf /var/lib/apt/lists/* + +RUN python3 -m pip install --no-cache-dir scapy==2.5.0 unittest-xml-reporting==3.2.0 && \ + python3 -m pip install --no-cache-dir --ignore-installed /opt/ptf + +RUN set -eux; \ + deb_dir=/opt/docker-sai-test-vpp/debs; \ + require_deb() { \ + local label="$1"; \ + shift; \ + local found=0; \ + local pattern; \ + for pattern in "$@"; do \ + if compgen -G "${deb_dir}/${pattern}" >/dev/null; then \ + found=1; \ + fi; \ + done; \ + if [[ "$found" -eq 0 ]]; then \ + echo "Missing ${label}; expected one of: $*" >&2; \ + exit 1; \ + fi; \ + }; \ + require_deb "VPP infra package" "libvppinfra_*.deb"; \ + require_deb "VPP package" "vpp_*.deb"; \ + require_deb "VPP core plugin package" "vpp-plugin-core_*.deb"; \ + require_deb "VPP DPDK plugin package" "vpp-plugin-dpdk_*.deb"; \ + require_deb "SAI virtual switch package" "libsaivs_*.deb"; \ + require_deb "SAI Redis package" "libsairedis_*.deb"; \ + require_deb "SAI metadata package" "libsaimetadata_*.deb"; \ + require_deb "SONiC SWSS common package" "libswsscommon_*.deb"; \ + require_deb "YANG runtime package" "libyang_*.deb" "libyang3_*.deb"; \ + require_deb "SAI thrift server package" "saiserver_*.deb" "saiserverv2_*.deb"; \ + require_deb "SAI Python thrift package" "python-saithrift_*.deb" "python-saithriftv2_*.deb"; \ + runtime_debs=(); \ + have_saiserverv2=0; \ + have_python_saithriftv2=0; \ + compgen -G "${deb_dir}/saiserverv2_*.deb" >/dev/null && have_saiserverv2=1; \ + compgen -G "${deb_dir}/python-saithriftv2_*.deb" >/dev/null && have_python_saithriftv2=1; \ + while IFS= read -r deb_file; do \ + deb_name="$(basename "$deb_file")"; \ + case "$deb_name" in \ + *-dbg_*.deb|*-dbgsym_*.deb|*-dev_*.deb) continue ;; \ + libyang3-tools_*.deb) continue ;; \ + saiserver_*.deb) [[ "$have_saiserverv2" -eq 1 ]] && continue ;; \ + python-saithrift_*.deb) [[ "$have_python_saithriftv2" -eq 1 ]] && continue ;; \ + esac; \ + runtime_debs+=("$deb_file"); \ + done < <(find "$deb_dir" -maxdepth 1 -type f -name '*.deb' | sort); \ + if [[ "${#runtime_debs[@]}" -eq 0 ]]; then \ + echo "No runtime .deb packages found under ${deb_dir}" >&2; \ + exit 1; \ + fi; \ + apt-get update; \ + cp /usr/sbin/sysctl /usr/sbin/sysctl.real; \ + cp /usr/bin/true /usr/sbin/sysctl; \ + apt-get install -y --no-install-recommends --allow-downgrades "${runtime_debs[@]}"; \ + find /usr/lib/python3/dist-packages -maxdepth 1 -type d -name 'saithrift-*.egg' -print > /usr/lib/python3/dist-packages/saithrift.pth; \ + test -s /usr/lib/python3/dist-packages/saithrift.pth; \ + python3 -c 'import sai_thrift'; \ + mv /usr/sbin/sysctl.real /usr/sbin/sysctl; \ + build_dev_debs=(); \ + for f in ${deb_dir}/libswsscommon-dev_*.deb; do \ + [[ -f "$f" ]] && build_dev_debs+=("$f"); \ + done; \ + if [[ ${#build_dev_debs[@]} -gt 0 ]]; then \ + apt-get install -y --no-install-recommends "${build_dev_debs[@]}"; \ + else \ + apt-get install -y --no-install-recommends libswsscommon-dev || true; \ + fi; \ + apt-get install -y --no-install-recommends g++ libhiredis-dev && \ + g++ -shared -fPIC -o /usr/local/lib/libswss_log_stdout.so \ + /opt/docker-sai-test-vpp/swss_log_stdout_preload.cpp -lswsscommon && \ + apt-get purge -y g++ libhiredis-dev && \ + if [[ ${#build_dev_debs[@]} -gt 0 ]]; then apt-get purge -y libswsscommon-dev; fi && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN install -d \ + /etc/sai \ + /run/vpp \ + /var/run/redis \ + /test-results \ + /usr/share/sonic/hwsku \ + /var/log && \ + install -m 0644 /opt/docker-sai-test-vpp/sai.profile /etc/sai/sai.profile && \ + install -m 0644 /opt/docker-sai-test-vpp/lanemap.ini /etc/sai/lanemap.ini && \ + install -m 0644 /opt/docker-sai-test-vpp/port-map.ini /etc/sai/port-map.ini && \ + install -m 0644 /opt/docker-sai-test-vpp/ptf-port-map.ini /etc/sai/ptf-port-map.ini && \ + install -m 0755 /opt/docker-sai-test-vpp/run_test.sh /usr/local/bin/run_test.sh && \ + ln -sf /usr/local/bin/run_test.sh /run_test.sh + +WORKDIR / + +# This is a single-purpose, --privileged test harness: the entrypoint starts VPP, +# Redis and saiserver, creates/destroys veth + PortChannel netdevs, and uses raw +# AF_PACKET sockets, all of which require root inside the container. It is run +# manually/in CI as a disposable test container, never as a deployed service, so a +# non-root USER would break the harness without adding meaningful isolation. +# nosemgrep: dockerfile.security.missing-user-entrypoint.missing-user-entrypoint +ENTRYPOINT ["/usr/local/bin/run_test.sh"] \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/README.md b/.azure-pipelines/docker-sai-test-vpp/README.md new file mode 100644 index 0000000000..a42262eb07 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/README.md @@ -0,0 +1,273 @@ +# docker-sai-test-vpp — VPP SAI Unit-Test Framework + +A self-contained, single-container framework for validating the **SAI API implementation of the VPP virtual-switch backend** (`libsaivs.so`) by running the OpenComputeProject (OCP) `sai_test` PTF suite against a real VPP dataplane. + +## a) Purpose and design + +### What it does + +The framework exercises SAI operations (create/set/get/remove of ports, RIFs, routes, neighbors, VLANs, FDB, LAGs, ECMP, …) through a Thrift RPC interface and verifies the resulting **data-plane** behavior by injecting and capturing packets on virtual interfaces. It is the test vehicle for finding gaps between the SAI contract and the VPP backend, and for producing a per-test **compatibility matrix**. + +### Architecture (one privileged container) + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ docker-sai-test-vpp (--privileged) │ +│ │ +│ ┌───────────┐ ┌──────────────┐ ┌────────────────────────┐ │ +│ │ VPP │ │ saiserver │ │ PTF test runner │ │ +│ │ af_packet │◄───►│ libsaivs.so │◄────►│ sai_test/*.py │ │ +│ │ linux_cp │ VPP │ (VPP SAI) │Thrift│ sai_thrift adapter │ │ +│ └─────┬─────┘ API └──────────────┘ :9092└───────────┬────────────┘ │ +│ │ AF_PACKET raw socket │ (AF_PACKET) │ +│ OEthernet0 ◄════ veth ════► OEth0_peer ──────────────┘ │ +│ … (32 pairs) │ +└──────────────────────────────────────────────────────────────────────┘ + EXISTING: VPP, libsaivs.so, sai_test, sai_thrift, PTF, af_packet/linux_cp + THIS HARNESS: Dockerfile, run_test.sh, sai.profile, *-map.ini +``` + +- **VPP** is the dataplane; `OEthernetX` are the VPP-facing kernel veth ends and `OEthX_peer` are the PTF-facing ends. `OEthernetX` represents an out-facing (wire) interface; the inside `EthernetX` (linux_cp TAP) faces the SONiC control plane. +- **saiserver** is a thin Thrift→SAI shim (port 9092) linked against `libsaivs.so`. +- **PTF** runs the OCP `sai_test` Python suite and does packet I/O on the `OEthX_peer` ends. +- **`run_test.sh`** is the container entrypoint and orchestrator: Redis → veth topology → VPP → saiserver → PTF. PortChannel netdevs and LAG/SVI connected IPs are set up in sai_test `setUp()` via sai_test's `SIMULATE_SONIC` helper (`config/simulate_sonic.py`), not in `run_test.sh`. + +### Key design point — config-signature grouping + +The VPP SAI backend can build the switch + host-interfaces **once per saiserver process**. The OCP framework is built to configure a common T0 setup once and have subsequent tests reuse it (`common_configured=true`). But different test classes ask `T0TestBase.setUp` for *different* common configs (e.g. ECMP tests need next-hop groups). So `run_test.sh`: + +1. parses each requested test class's `setUp` (resolving config kwargs through the class **inheritance chain**) to compute its common-config "signature"; +2. groups tests by signature; +3. for each group: **restarts the backend** (fresh saiserver), the first test builds + persists that group's config, the rest reuse it. + +This lets one container run any mix of tests correctly in a single invocation. + +### Supported tests + +The table below lists OCP `sai_test` classes that **pass** on the current VPP SAI backend (last strict validation **2026-07-21** against `sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test`). It is the published substitute for a full compatibility matrix: only passing tests are listed. After a local matrix run, update this section when the pass set changes (see **Collecting results** below). + +| Module | Passing test classes | +|---|---| +| `sai_ecmp_test` | `EcmpLagDisableTestV4`, `EcmpLagDisableTestV6`, `EcmpReuseLagRouteV4`, `EcmpReuseLagRouteV6`, `RemoveAllNextHopMemeberTestV4`, `RemoveNexthopGroupTestV4` | +| `sai_neighbor_test` | `AddHostRouteTestV6`, `NhopDiffPrefixRemoveLonger`, `NhopDiffPrefixRemoveLongerV6`, `NhopDiffPrefixRemoveShorter`, `NhopDiffPrefixRemoveShorterV6` | +| `sai_rif_test` | — | +| `sai_route_test` | `LagMultipleRoutev6Test`, `RemoveRouteV4Test`, `RouteDiffPrefixAddThenDeleteLongerV4Test`, `RouteDiffPrefixAddThenDeleteLongerV6Test`, `RouteDiffPrefixAddThenDeleteShorterV4Test`, `RouteDiffPrefixAddThenDeleteShorterV6Test`, `RouteSameSipDipv6Test`, `StaicSviMacFloodingTest` | + +**19** classes passing (of 91 JUnit rows in the last strict full matrix run). + +## b) Building the framework + +The image bundles pre-built `.deb` packages from `debs/` (git-ignored locally). You only need to regenerate `.deb`s when the corresponding source changes. + +### Use cases + +The framework is designed to support three distinct deployment and testing scenarios: + +#### **Use case 1: Local dev in a `sonic-buildimage` workspace** +- **What changes:** `vslib/` C++ backend, `SAI/test/sai_test` Python tests, VPP packages, or harness scripts. +- **How dependencies are supplied:** Copy runtime `.deb`s from `target/debs//` into `docker-sai-test-vpp/debs/`, then build the image. The default suite today is **trixie** (see Dockerfile). +- **Status:** Supported today via `build_harness.sh` (below). Assumes the workspace layout `sonic-buildimage/src/sonic-sairedis`. VPP packages can come from `target/debs/trixie/` after a platform-vpp build, from a `sonic-platform-vpp` pipeline download, or from `VPP_DEB_DIR` when running the build script. + +#### **Use case 2: `sonic-sairedis` PR CI** +- **What changes:** `vslib/` C++ backend, harness files, or OCP tests. +- **How dependencies are supplied:** The pipeline's **Build** stage compiles and produces fresh `libsairedis` / `libsaivs` / `saiserver` / `python-saithrift` artifacts. Other runtime `.deb`s (`libswsscommon`, `libyang`, VPP) must be downloaded from existing pipeline artifacts — following the same pattern as `.azure-pipelines/build-docker-sonic-vs-template.yml` (swss-common pipeline, sonic-platform-vpp `vpp-trixie`, buildimage common libs). +- **Status:** Documented intent; CI wiring is follow-up work for this PR (Phase 3). +- **Required runtime packages and typical artifact sources:** + +| Package glob | PR build produces? | Typical CI download source | +|---|---|---| +| `libsaivs_*`, `libsairedis_*`, `libsaimetadata_*`, `saiserverv2_*`, `python-saithriftv2_*` | Yes (this repo's Build stage) | Current pipeline job artifacts | +| `libswsscommon_*` | No | `Azure.sonic-swss-common` pipeline artifact | +| `libyang_*` | No | `sonic-buildimage` common-lib / VS build artifact | +| `libvppinfra_*`, `vpp_*`, `vpp-plugin-*` | No | `sonic-net.sonic-platform-vpp` artifact `vpp-trixie` | + +#### **Use case 3: `sonic-platform-vpp` PR CI** +- **What changes:** VPP `.deb`s only. +- **How dependencies are supplied:** Pull a pre-built `docker-sai-test-vpp` image from the `sonic-sairedis` pipeline artifact, then rebuild/replace only the VPP packages inside `debs/` (or `docker build` with updated VPP debs on top of the cached layers). +- **Status:** Documented intent; requires Use Case 2 image publish first. +- **Workflow:** After the harness image is published as a pipeline artifact in Use Case 2, a platform-vpp job can `docker load` that image, overlay freshly built VPP `.deb`s into `debs/`, and rebuild only the package-install layer (or run tests in a container with VPP packages swapped in). + +### Required `.deb` packages (validated by the Dockerfile) + +| Package glob | Source repo / how produced | +|---|---| +| `libvppinfra_*`, `vpp_*`, `vpp-plugin-core_*`, `vpp-plugin-dpdk_*` | VPP packages (from the `sonic-platform-vpp` build) | +| `libsaivs_*` | `sonic-sairedis` — the VPP SAI backend under test | +| `libsairedis_*`, `libsaimetadata_*` | `sonic-sairedis` | +| `libswsscommon_*` | `sonic-swss-common` | +| `libyang_*` / `libyang3_*` | `sonic-buildimage` (libyang3 on trixie) | +| `saiserver_*` / `saiserverv2_*` | `sonic-sairedis` SAI Thrift server | +| `python-saithrift_*` / `python-saithriftv2_*` | `sonic-sairedis` SAI Thrift Python client | + +All of these are staged in [`debs/`](debs/) (local-only). The Dockerfile installs every runtime `.deb` it finds there (skipping `-dbg`/`-dev`/`-dbgsym`). + +### Build script (use case 1) + +`build_harness.sh` automates staging from `sonic-buildimage` and building the image. It validates that all required runtime `.deb`s are present in `debs/` before `docker build`. If sonic-sairedis packages are missing, it runs the trixie `make` targets automatically (disable with `--no-auto-build`). VPP, `libswsscommon`, and `libyang`/`libyang3` are not auto-built — the script fails with hints if they are absent. + +```bash +cd /src/sonic-sairedis/.azure-pipelines/docker-sai-test-vpp +./build_harness.sh +``` + +Useful options: `--build-sairedis` (force-rebuild sairedis debs even when present), `--no-auto-build` (never invoke `make` for missing sairedis debs), `--no-stage-debs` (image rebuild only), `--vpp-deb-dir `, `--image-tag `. Run `./build_harness.sh --help` for the full list. + +### Regenerating the SAI `.deb`s manually (only when `vslib/` C++ changes) + +From the **buildimage repo root** (`sonic-buildimage`): + +```bash +cd + +# Force re-generation by removing the stale targets first +rm -f target/debs/trixie/libsairedis_*.deb target/debs/trixie/libsairedis-dev_*.deb \ + target/debs/trixie/libsaivs_*.deb target/debs/trixie/libsaivs-dev_*.deb + +# Build libsaivs / libsairedis +make target/debs/trixie/libsairedis_1.0.0_amd64.deb + +# Build saiserver + saithrift client (if those changed) +BLDENV=trixie make -f Makefile.work target/debs/trixie/libsaithrift-dev_0.9.4_amd64.deb + +# Stage the fresh packages into the harness build context +cp target/debs/trixie/libsaivs_*.deb target/debs/trixie/libsaivs-dev_*.deb \ + target/debs/trixie/libsairedis_*.deb target/debs/trixie/libsairedis-dev_*.deb \ + target/debs/trixie/saiserver_*.deb target/debs/trixie/python-saithrift_*.deb \ + src/sonic-sairedis/.azure-pipelines/docker-sai-test-vpp/debs/ +``` + +Then run `./build_harness.sh` (or `./build_harness.sh --no-stage-debs` if `debs/` is already up to date). + +> Behind a corporate proxy, prefix `make` with your Docker build credentials, e.g. `DOCKER_CONFIG=`. VPP `.deb`s come from the `sonic-platform-vpp` pipeline; drop new ones into `debs/` or pass `--vpp-deb-dir`. `build_harness.sh` forwards proxy env vars to `docker build`. + +### Building the image manually + +Equivalent to `./build_harness.sh --no-stage-debs` after `debs/` is populated. From the **`sonic-sairedis`** directory: + +```bash +cd /src/sonic-sairedis + +docker build --no-cache \ + -f .azure-pipelines/docker-sai-test-vpp/Dockerfile \ + -t docker-sai-test-vpp:phase1 . +``` + +If you are behind a proxy, pass it through (both lower- and upper-case): + +```bash +docker build --no-cache \ + --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy \ + --build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$https_proxy \ + --build-arg no_proxy=$no_proxy --build-arg NO_PROXY=$no_proxy \ + -f .azure-pipelines/docker-sai-test-vpp/Dockerfile \ + -t docker-sai-test-vpp:phase1 . +``` + +> Rebuild scope: editing only `run_test.sh` / `sai_test/**` needs an **image rebuild** only (fast). Editing `vslib/` C++ needs a **`.deb` rebuild** (above) first, then the image. + +## c) Running tests + +The container entrypoint is `run_test.sh`. Test selectors are PTF targets: `module` (e.g. `sai_route_test`) or `module.Class` (e.g. `sai_route_test.RouteRifTest`). `PORT_COUNT` sets the port/veth count (use 32 for the standard T0 topology). + +### Where the tests come from + +The OCP `sai_test` suite is baked into the image at `/sai_test` (copied from `SAI/test/sai_test/` in the repo). PTF and the SAI Thrift client come from `SAI/test/ptf` and the `python-saithrift` package. `run_test.sh` discovers test classes under `/sai_test` automatically. + +### Run a single test + +```bash +docker run --rm --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:phase1 sai_route_test.RouteRifTest +``` + +### Run several tests (auto-grouped by config, one container) + +```bash +docker run --rm --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:phase1 \ + sai_route_test.RouteRifTest sai_ecmp_test.EcmpHashFieldSportTestV4 +``` + +### Run whole modules, or everything + +```bash +# whole modules +docker run --rm --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:phase1 sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test + +# every discovered test class (no args) +docker run --rm --privileged -e PORT_COUNT=32 docker-sai-test-vpp:phase1 +``` + +### Collecting results (JUnit XML) and updating the supported-test list + +PTF writes one JUnit-XML file per test into `/test-results`. By convention these land in the local-only `results/` tree (git-ignored; not published to the remote). Run from the `docker-sai-test-vpp/` directory and bind-mount `results/xml` out: + +```bash +cd /src/sonic-sairedis/.azure-pipelines/docker-sai-test-vpp +mkdir -p results/xml +rm -f results/xml/TEST-*.xml +docker run --rm --privileged -e PORT_COUNT=32 \ + -v "$PWD/results/xml:/test-results" \ + docker-sai-test-vpp:phase1 \ + sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test \ + 2>&1 | tee results/run.log +``` + +Then build a local compatibility matrix with the bundled generator (defaults to the `results/` tree). The generator parses JUnit XML with `defusedxml` (hardened against XXE), so install it once if needed: + +```bash +pip install defusedxml +python3 gen_compatibility_matrix.py # writes results/compatibility-matrix.md +``` + +`gen_compatibility_matrix.py` walks `results/xml/TEST-*.xml` and writes a PASS/FAIL/ERROR/SKIP table with a count summary. You can also pass an explicit ` [output.md]` to point it elsewhere. + +**Publishing pass results:** when the set of passing tests changes, update the **Supported tests** section in this `README.md` from the local matrix (list only classes with PASS; do not commit the matrix itself). Working notes and deep-dive logs may be kept under `devdocs/` (also local-only and git-ignored). + +## d) Additional information + +Debug hold mode, environment variables, in-container log paths, and common pitfalls. + +### Debug mode (leave VPP / saiserver / veths alive for inspection) + +```bash +docker rm -f officesai-debug 2>/dev/null +docker run -d --name officesai-debug --privileged -e PORT_COUNT=32 \ + docker-sai-test-vpp:phase1 --debug sai_route_test.RouteRifTest + +# while the test runs, inspect VPP state: +docker exec officesai-debug vppctl show interface +docker exec officesai-debug vppctl show ip fib +docker exec officesai-debug vppctl show bond +docker cp officesai-debug:/var/log/saiserver.log ./saiserver.log +``` + +In `--debug` the container leaves the dataplane running after the test so you can use `vppctl`; remember to `docker rm -f officesai-debug` when done. + +### Environment knobs + +| Variable | Default | Meaning | +|---|---|---| +| `PORT_COUNT` | 32 | number of `OEthernetX`/`OEthX_peer` veth pairs | +| `COMMON_CONFIGURED_REUSE` | 1 | 1 = config-signature grouping + reuse; 0 = legacy single ptf invocation | +| `KEEP_VETHS_UP_SECONDS` | 120 | how long the per-group watchdog keeps VPP host-interfaces/veths up | +| `LAG_RIF_IPS` | 1 | enable LAG RIF connected-IP assignment in sai_test setUp (`SIMULATE_SONIC`) | +| `SVI_RIF_IPS` | 1 | enable SVI RIF connected-IP assignment in sai_test setUp | +| `SIMULATE_SONIC` | 1 | set by `run_test.sh`; enables sai_test's SONiC control-plane simulation (PortChannel netdevs + LAG/SVI RIF IPs) | +| `TEST_FILTER` | — | alternative way to pass a single selector via env | + +These are read by `run_test.sh` at container start (defaults shown). + +### Logs inside the container + +- `/var/log/saiserver.log` — saiserver stdout/stderr **plus** `libsaivs` `SWSS_LOG_*` lines. After the SAI change that removed `swss::Logger` setup from `saiserver.cpp`, the harness routes `SWSS_LOG_*` to stdout via an `LD_PRELOAD` shim (`swss_log_stdout_preload.cpp` → `/usr/local/lib/libswss_log_stdout.so`) so SAI backend traces still land in this file. `sai_log_set()` API-level notices from saiserver itself are also configured there. +- `/var/log/vpp.log`, `/var/log/vpp-startup.log` — VPP CLI history / stdout (crash backtraces). +- `/var/log/vpp-api-trace.txt` — decoded VPP binary-API trace (dumped at teardown). +- `/test-results/TEST-*.xml` — per-test JUnit results (bind-mount to the local `results/xml/` directory on the host). + +### Gotchas + +- The container must be `--privileged` (raw AF_PACKET sockets on veths/TAPs). +- The VPP SAI backend can build the switch once per saiserver process; the per-group backend restart in `run_test.sh` handles this automatically — do not expect to re-run a full config build inside a single long-lived saiserver. +- A benign `buffer: numa[1] falling back to non-hugepage backed buffer pool` line at teardown is a host hugepage-availability warning, not a test failure. diff --git a/.azure-pipelines/docker-sai-test-vpp/build_harness.sh b/.azure-pipelines/docker-sai-test-vpp/build_harness.sh new file mode 100755 index 0000000000..bc5d1bfc2c --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/build_harness.sh @@ -0,0 +1,317 @@ +#!/usr/bin/env bash +# Stage runtime .deb packages and build the docker-sai-test-vpp image. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SAIREDIS_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +BUILDIMAGE_ROOT="${SONIC_BUILDIMAGE:-$(cd "${SAIREDIS_ROOT}/../.." && pwd)}" +DEB_STAGING="${SCRIPT_DIR}/debs" +DOCKERFILE="${SAIREDIS_ROOT}/.azure-pipelines/docker-sai-test-vpp/Dockerfile" +BLDENV="${BLDENV:-trixie}" +IMAGE_TAG="${IMAGE_TAG:-docker-sai-test-vpp:phase1}" +STAGE_DEBS="${STAGE_DEBS:-1}" +BUILD_SAIREDIS_DEBS="${BUILD_SAIREDIS_DEBS:-0}" +AUTO_BUILD_SAIREDIS_DEBS="${AUTO_BUILD_SAIREDIS_DEBS:-1}" +VPP_DEB_DIR="${VPP_DEB_DIR:-}" + +SAIREDIS_DEB_PATTERNS=( + 'libsaivs_*.deb' 'libsairedis_*.deb' 'libsaimetadata_*.deb' + 'saiserver_*.deb' 'saiserverv2_*.deb' + 'python-saithrift_*.deb' 'python-saithriftv2_*.deb' +) + +usage() +{ + cat <<'EOF' +Usage: build_harness.sh [options] + +Stage runtime .deb packages into docker-sai-test-vpp/debs/ (local-only, +git-ignored) and build the test image from the sonic-sairedis repo root. + +Options: + --no-stage-debs Skip copying .debs (use what is already in debs/) + --build-sairedis Force-rebuild libsairedis/libsaivs (+ saithrift) in + sonic-buildimage before staging + --no-auto-build Do not invoke make when sonic-sairedis .debs are + missing (validate only; fail with hints) + --bldenv Debian suite for target/debs (default: trixie) + --image-tag Docker image tag (default: docker-sai-test-vpp:phase1) + --vpp-deb-dir Directory of VPP .debs (default: search buildimage + target/debs/ and VPP_DEB_DIR env) + -h, --help Show this help + +Environment: + SONIC_BUILDIMAGE Path to sonic-buildimage root (auto-detected if unset) + VPP_DEB_DIR Extra directory to copy VPP .debs from + AUTO_BUILD_SAIREDIS_DEBS 1 (default): build sairedis .debs when missing + http_proxy/https_proxy/no_proxy Passed through to docker build when set + +Examples: + # Use-case 1 (local dev): stage, auto-build sairedis if needed, build image + ./build_harness.sh + + # Force-rebuild vslib .debs, then stage + build + ./build_harness.sh --build-sairedis + + # Rebuild image only (debs/ already populated and complete) + ./build_harness.sh --no-stage-debs +EOF +} + +log() +{ + printf '[build_harness] %s\n' "$*" +} + +die() +{ + printf '[build_harness] ERROR: %s\n' "$*" >&2 + exit 1 +} + +deb_present() +{ + local dir="$1" + shift + local pattern + + [[ -d "$dir" ]] || return 1 + + for pattern in "$@"; do + shopt -s nullglob + local matches=("${dir}"/${pattern}) + shopt -u nullglob + if [[ ${#matches[@]} -gt 0 ]]; then + return 0 + fi + done + + return 1 +} + +copy_matching_debs() +{ + local src_dir="$1" + shift + local pattern + + [[ -d "$src_dir" ]] || return 0 + + for pattern in "$@"; do + shopt -s nullglob + local matches=("${src_dir}"/${pattern}) + shopt -u nullglob + if [[ "${#matches[@]}" -eq 0 ]]; then + continue + fi + cp -v "${matches[@]}" "${DEB_STAGING}/" + done +} + +stage_vpp_debs() +{ + local src_dir="$1" + copy_matching_debs "${src_dir}" \ + 'libvppinfra_*.deb' 'vpp_*.deb' 'vpp-plugin-core_*.deb' 'vpp-plugin-dpdk_*.deb' +} + +stage_sairedis_debs_from_buildimage() +{ + local deb_dir="${BUILDIMAGE_ROOT}/target/debs/${BLDENV}" + + [[ -d "${BUILDIMAGE_ROOT}" ]] || die "sonic-buildimage not found at ${BUILDIMAGE_ROOT}" + mkdir -p "${DEB_STAGING}" + + log "Staging sonic-sairedis .debs from ${deb_dir} into ${DEB_STAGING}" + copy_matching_debs "${deb_dir}" "${SAIREDIS_DEB_PATTERNS[@]}" +} + +stage_debs_from_buildimage() +{ + local deb_dir="${BUILDIMAGE_ROOT}/target/debs/${BLDENV}" + + [[ -d "${BUILDIMAGE_ROOT}" ]] || die "sonic-buildimage not found at ${BUILDIMAGE_ROOT}" + mkdir -p "${DEB_STAGING}" + + log "Staging runtime .debs from ${deb_dir} into ${DEB_STAGING}" + + stage_vpp_debs "${deb_dir}" + copy_matching_debs "${deb_dir}" \ + 'libsaivs_*.deb' 'libsairedis_*.deb' 'libsaimetadata_*.deb' \ + 'libswsscommon_*.deb' 'libswsscommon-dev_*.deb' \ + 'libyang_*.deb' 'libyang3_*.deb' 'libpcre3_*.deb' \ + 'saiserver_*.deb' 'saiserverv2_*.deb' \ + 'python-saithrift_*.deb' 'python-saithriftv2_*.deb' + + if [[ -n "${VPP_DEB_DIR}" ]]; then + log "Staging VPP .debs from VPP_DEB_DIR=${VPP_DEB_DIR}" + stage_vpp_debs "${VPP_DEB_DIR}" + fi +} + +sairedis_debs_satisfied() +{ + deb_present "${DEB_STAGING}" 'libsaivs_*.deb' \ + && deb_present "${DEB_STAGING}" 'libsairedis_*.deb' \ + && deb_present "${DEB_STAGING}" 'libsaimetadata_*.deb' \ + && deb_present "${DEB_STAGING}" 'saiserver_*.deb' 'saiserverv2_*.deb' \ + && deb_present "${DEB_STAGING}" 'python-saithrift_*.deb' 'python-saithriftv2_*.deb' +} + +build_sairedis_debs() +{ + local force="${1:-0}" + + [[ -d "${BUILDIMAGE_ROOT}" ]] || die "sonic-buildimage not found at ${BUILDIMAGE_ROOT}" + + if [[ "${force}" == "1" ]]; then + log "Force-rebuilding libsairedis/libsaivs in ${BUILDIMAGE_ROOT} (BLDENV=${BLDENV})" + ( + cd "${BUILDIMAGE_ROOT}" + rm -f "target/debs/${BLDENV}/libsairedis_"*.deb "target/debs/${BLDENV}/libsairedis-dev_"*.deb \ + "target/debs/${BLDENV}/libsaivs_"*.deb "target/debs/${BLDENV}/libsaivs-dev_"*.deb + ) + else + log "Building libsairedis/libsaivs in ${BUILDIMAGE_ROOT} (BLDENV=${BLDENV})" + fi + + ( + cd "${BUILDIMAGE_ROOT}" + if [[ "${BLDENV}" == bookworm ]]; then + NOTRIXIE=1 make "target/debs/${BLDENV}/libsairedis_1.0.0_amd64.deb" + NOTRIXIE=1 BLDENV="${BLDENV}" make -f Makefile.work \ + "target/debs/${BLDENV}/libsaithrift-dev_0.9.4_amd64.deb" + else + make "target/debs/${BLDENV}/libsairedis_1.0.0_amd64.deb" + BLDENV="${BLDENV}" make -f Makefile.work \ + "target/debs/${BLDENV}/libsaithrift-dev_0.9.4_amd64.deb" + fi + ) +} + +report_missing_debs() +{ + local -a hints=() + + if ! sairedis_debs_satisfied; then + hints+=( + "sonic-sairedis (libsaivs, libsairedis, libsaimetadata, saiserverv2, python-saithriftv2):" + " ./build_harness.sh --build-sairedis" + " (or omit --no-auto-build to build automatically when missing)" + ) + fi + + if ! deb_present "${DEB_STAGING}" 'libvppinfra_*.deb'; then + hints+=( + "VPP infra (libvppinfra_*.deb):" + " Build platform-vpp in sonic-buildimage, or download sonic-net.sonic-platform-vpp" + " pipeline artifact vpp-trixie, then re-run with --vpp-deb-dir or VPP_DEB_DIR" + ) + fi + if ! deb_present "${DEB_STAGING}" 'vpp_*.deb'; then + hints+=( + "VPP (vpp_*.deb): same sources as libvppinfra (platform-vpp build or vpp-trixie artifact)" + ) + fi + if ! deb_present "${DEB_STAGING}" 'vpp-plugin-core_*.deb'; then + hints+=( + "VPP core plugin (vpp-plugin-core_*.deb): same sources as libvppinfra" + ) + fi + if ! deb_present "${DEB_STAGING}" 'vpp-plugin-dpdk_*.deb'; then + hints+=( + "VPP DPDK plugin (vpp-plugin-dpdk_*.deb): same sources as libvppinfra" + ) + fi + if ! deb_present "${DEB_STAGING}" 'libswsscommon_*.deb'; then + hints+=( + "SONiC SWSS common (libswsscommon_*.deb):" + " From sonic-buildimage: make target/debs/${BLDENV}/libswsscommon_1.0.0_amd64.deb" + " Or download Azure.sonic-swss-common pipeline artifact, then re-run ./build_harness.sh" + ) + fi + if ! deb_present "${DEB_STAGING}" 'libyang_*.deb' 'libyang3_*.deb'; then + hints+=( + "YANG runtime (libyang_*.deb or libyang3_*.deb):" + " From sonic-buildimage: make target/debs/${BLDENV}/libyang3_3.12.2-1_amd64.deb" + " Or download sonic-buildimage.common_libs / VS build artifact, then re-run ./build_harness.sh" + ) + fi + + if [[ ${#hints[@]} -eq 0 ]]; then + return 0 + fi + + printf '[build_harness] ERROR: Missing required .debs in %s\n' "${DEB_STAGING}" >&2 + local line + for line in "${hints[@]}"; do + printf ' %s\n' "$line" >&2 + done + return 1 +} + +ensure_staged_debs() +{ + mkdir -p "${DEB_STAGING}" + + if ! sairedis_debs_satisfied && [[ "${AUTO_BUILD_SAIREDIS_DEBS}" == "1" ]] \ + && [[ "${BUILD_SAIREDIS_DEBS}" != "1" ]]; then + log "sonic-sairedis .debs missing; auto-building in sonic-buildimage" + build_sairedis_debs 0 + stage_sairedis_debs_from_buildimage + fi + + report_missing_debs || die "cannot build image until required .debs are in ${DEB_STAGING}" +} + +docker_build_args() +{ + local -a args=(docker build --no-cache -f "${DOCKERFILE}" -t "${IMAGE_TAG}" .) + local var + + for var in http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY; do + if [[ -n "${!var:-}" ]]; then + args+=(--build-arg "${var}=${!var}") + fi + done + + printf '%s\0' "${args[@]}" +} + +main() +{ + while [[ $# -gt 0 ]]; do + case "$1" in + --no-stage-debs) STAGE_DEBS=0 ;; + --build-sairedis) BUILD_SAIREDIS_DEBS=1 ;; + --no-auto-build) AUTO_BUILD_SAIREDIS_DEBS=0 ;; + --bldenv) shift; BLDENV="${1:?--bldenv requires a value}" ;; + --image-tag) shift; IMAGE_TAG="${1:?--image-tag requires a value}" ;; + --vpp-deb-dir) shift; VPP_DEB_DIR="${1:?--vpp-deb-dir requires a value}" ;; + -h|--help) usage; exit 0 ;; + *) die "unknown option: $1 (try --help)" ;; + esac + shift + done + + if [[ "${BUILD_SAIREDIS_DEBS}" == "1" ]]; then + build_sairedis_debs 1 + stage_sairedis_debs_from_buildimage + fi + + if [[ "${STAGE_DEBS}" == "1" ]]; then + stage_debs_from_buildimage + fi + + ensure_staged_debs + + log "Building image ${IMAGE_TAG} from ${SAIREDIS_ROOT}" + ( + cd "${SAIREDIS_ROOT}" + readarray -d '' -t docker_args < <(docker_build_args) + "${docker_args[@]}" + ) + log "Done: ${IMAGE_TAG}" +} + +main "$@" diff --git a/.azure-pipelines/docker-sai-test-vpp/gen_compatibility_matrix.py b/.azure-pipelines/docker-sai-test-vpp/gen_compatibility_matrix.py new file mode 100755 index 0000000000..9a93c47e4a --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/gen_compatibility_matrix.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +""" +Generate a Markdown compatibility matrix from PTF JUnit-XML results. + +The VPP SAI unit-test harness (run_test.sh) writes one JUnit-XML file per test +into its --xunit-dir (mounted out as /test-results -> a host directory). By +convention that host directory is this harness's own results tree: + + docker-sai-test-vpp/results/xml/ <- per-test TEST-*.xml + docker-sai-test-vpp/results/compatibility-matrix.md <- generated matrix + +This script walks the TEST-*.xml files and emits a PASS/FAIL/ERROR/SKIP table. + +Usage: + # use the default results tree next to this script + python3 gen_compatibility_matrix.py + + # or point at an explicit xml dir / output file + python3 gen_compatibility_matrix.py [output.md] + +Example workflow: + cd docker-sai-test-vpp + mkdir -p results/xml + docker run --rm --privileged -e PORT_COUNT=32 \ + -v "$PWD/results/xml:/test-results" \ + docker-sai-test-vpp:phase1 \ + sai_route_test sai_rif_test sai_neighbor_test sai_ecmp_test \ + 2>&1 | tee results/run.log + python3 gen_compatibility_matrix.py # writes results/compatibility-matrix.md +""" + +import sys +import os +import glob +import datetime + +# Use defusedxml (hardened against XXE / entity-expansion attacks) to parse the +# PTF JUnit XML. Install with `pip install defusedxml` if it is not already present. +import defusedxml.ElementTree as ET + +# Default results tree lives alongside this script, under docker-sai-test-vpp/. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_RESULTS_DIR = os.path.join(_SCRIPT_DIR, "results") +DEFAULT_XML_DIR = os.path.join(DEFAULT_RESULTS_DIR, "xml") +DEFAULT_OUTPUT = os.path.join(DEFAULT_RESULTS_DIR, "compatibility-matrix.md") + +ICON = { + "PASS": "\u2705 PASS", + "FAIL": "\u274c FAIL", + "ERROR": "\u26a0\ufe0f ERROR", + "SKIP": "\u23ed\ufe0f SKIP", +} + +# Result-status key (the icons used in the Result column). +RESULT_LEGEND = [ + ("\u2705 PASS", "test passed"), + ("\u274c FAIL", "assertion failed (e.g. expected packet not received)"), + ("\u26a0\ufe0f ERROR", "test errored out (exception / setUp / tearDown)"), + ("\u23ed\ufe0f SKIP", "test was skipped"), +] + +# SAI status codes that commonly appear in the Detail column. Values mirror +# SAI/inc/saistatus.h so a failing `status == -N` is readable at a glance. +SAI_STATUS_LEGEND = [ + ("0", "SAI_STATUS_SUCCESS"), + ("-1", "SAI_STATUS_FAILURE"), + ("-2", "SAI_STATUS_NOT_SUPPORTED"), + ("-3", "SAI_STATUS_NO_MEMORY"), + ("-4", "SAI_STATUS_INSUFFICIENT_RESOURCES"), + ("-5", "SAI_STATUS_INVALID_PARAMETER"), + ("-6", "SAI_STATUS_ITEM_ALREADY_EXISTS"), + ("-7", "SAI_STATUS_ITEM_NOT_FOUND"), + ("-8", "SAI_STATUS_BUFFER_OVERFLOW"), + ("-9", "SAI_STATUS_INVALID_PORT_NUMBER"), + ("-10", "SAI_STATUS_INVALID_PORT_MEMBER"), + ("-11", "SAI_STATUS_INVALID_VLAN_ID"), + ("-12", "SAI_STATUS_UNINITIALIZED"), + ("-13", "SAI_STATUS_TABLE_FULL"), + ("-14", "SAI_STATUS_MANDATORY_ATTRIBUTE_MISSING"), + ("-15", "SAI_STATUS_NOT_IMPLEMENTED"), + ("-16", "SAI_STATUS_ADDR_NOT_FOUND"), + ("-17", "SAI_STATUS_OBJECT_IN_USE"), +] + + +def collect_rows(xml_dir): + rows = [] + for path in sorted(glob.glob(os.path.join(xml_dir, "TEST-*.xml"))): + try: + root = ET.parse(path).getroot() + except Exception as e: # malformed XML -> surface, don't crash + rows.append(("?", os.path.basename(path), "PARSE-ERR", str(e)[:60])) + continue + suites = [root] if root.tag == "testsuite" else root.iter("testsuite") + for suite in suites: + for tc in suite.iter("testcase"): + mod = tc.get("classname", "") + name = tc.get("name", "") + fail = tc.find("failure") + err = tc.find("error") + skip = tc.find("skipped") + if err is not None: + status, detail = "ERROR", (err.get("message") or "") + elif fail is not None: + status, detail = "FAIL", (fail.get("message") or "") + elif skip is not None: + status, detail = "SKIP", (skip.get("message") or "") + else: + status, detail = "PASS", "" + detail = detail.replace("\n", " ").strip()[:80] + rows.append((mod, name, status, detail)) + rows.sort() + return rows + + +def render(rows): + counts = {} + for r in rows: + counts[r[2]] = counts.get(r[2], 0) + 1 + total = len(rows) + out = [] + out.append("# VPP SAI Compatibility Matrix\n") + generated = datetime.datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + out.append(f"_Generated: {generated}_\n") + + # Summary table: one row per status (in a stable, meaningful order) plus a + # total, with counts and percentages. + out.append("## Summary\n") + out.append("| Result | Count | % |") + out.append("|--------|------:|----:|") + order = ["PASS", "FAIL", "ERROR", "SKIP"] + seen = set(order) + for status in order + [s for s in sorted(counts) if s not in seen]: + n = counts.get(status, 0) + if n == 0 and status in seen: + continue + pct = (100.0 * n / total) if total else 0.0 + out.append(f"| {ICON.get(status, status)} | {n} | {pct:.1f}% |") + out.append(f"| **Total** | **{total}** | **100.0%** |") + out.append("") + + # Legend: result-status icons + SAI status codes seen in the Detail column. + out.append("## Legend\n") + out.append("**Result status**\n") + out.append("| Icon | Meaning |") + out.append("|------|---------|") + for icon, meaning in RESULT_LEGEND: + out.append(f"| {icon} | {meaning} |") + out.append("") + out.append("**SAI status codes** (see `SAI/inc/saistatus.h`) \u2014 appear in the Detail column\n") + out.append("| Code | Symbol |") + out.append("|------|--------|") + for code, symbol in SAI_STATUS_LEGEND: + out.append(f"| `{code}` | `{symbol}` |") + out.append("") + + out.append("## Results\n") + out.append("| Module | Test Class | Result | Detail |") + out.append("|--------|------------|--------|--------|") + for mod, name, status, detail in rows: + out.append(f"| `{mod}` | `{name}` | {ICON.get(status, status)} | {detail} |") + return "\n".join(out) + + +def main(argv): + args = argv[1:] + if args and args[0] in ("-h", "--help"): + sys.stderr.write(__doc__) + return 0 + + xml_dir = args[0] if len(args) >= 1 else DEFAULT_XML_DIR + # output: explicit 2nd arg, else default file when using the default xml dir, + # else stdout (so piping `> file.md` still works for a custom dir). + if len(args) >= 2: + output = args[1] + elif len(args) == 0: + output = DEFAULT_OUTPUT + else: + output = None # custom xml dir, no output given -> stdout + + if not os.path.isdir(xml_dir): + sys.stderr.write( + f"error: xml dir not found: {xml_dir}\n" + f" run the harness with -v /results/xml:/test-results first.\n" + ) + return 2 + + matrix = render(collect_rows(xml_dir)) + if output: + os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) + with open(output, "w", encoding="utf-8") as f: + f.write(matrix + "\n") + sys.stderr.write(f"wrote {output}\n") + else: + print(matrix) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.azure-pipelines/docker-sai-test-vpp/lanemap.ini b/.azure-pipelines/docker-sai-test-vpp/lanemap.ini new file mode 100644 index 0000000000..b9ec948707 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/lanemap.ini @@ -0,0 +1,34 @@ +# VPP wire-side interface to SAI lane mapping. +# Lanes match the default sai_test resources/config_db.json (Ethernet0,4,8,...,124). +OEthernet0:65,66,67,68 +OEthernet1:69,70,71,72 +OEthernet2:73,74,75,76 +OEthernet3:77,78,79,80 +OEthernet4:33,34,35,36 +OEthernet5:37,38,39,40 +OEthernet6:41,42,43,44 +OEthernet7:45,46,47,48 +OEthernet8:49,50,51,52 +OEthernet9:53,54,55,56 +OEthernet10:57,58,59,60 +OEthernet11:61,62,63,64 +OEthernet12:81,82,83,84 +OEthernet13:85,86,87,88 +OEthernet14:89,90,91,92 +OEthernet15:93,94,95,96 +OEthernet16:97,98,99,100 +OEthernet17:101,102,103,104 +OEthernet18:105,106,107,108 +OEthernet19:109,110,111,112 +OEthernet20:1,2,3,4 +OEthernet21:5,6,7,8 +OEthernet22:9,10,11,12 +OEthernet23:13,14,15,16 +OEthernet24:17,18,19,20 +OEthernet25:21,22,23,24 +OEthernet26:25,26,27,28 +OEthernet27:29,30,31,32 +OEthernet28:113,114,115,116 +OEthernet29:117,118,119,120 +OEthernet30:121,122,123,124 +OEthernet31:125,126,127,128 \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/port-map.ini b/.azure-pipelines/docker-sai-test-vpp/port-map.ini new file mode 100644 index 0000000000..726baadac3 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/port-map.ini @@ -0,0 +1,34 @@ +# saiserver front-panel alias to SAI lane mapping. +# Port names and lanes match the default sai_test resources/config_db.json. +Ethernet0 65,66,67,68 +Ethernet4 69,70,71,72 +Ethernet8 73,74,75,76 +Ethernet12 77,78,79,80 +Ethernet16 33,34,35,36 +Ethernet20 37,38,39,40 +Ethernet24 41,42,43,44 +Ethernet28 45,46,47,48 +Ethernet32 49,50,51,52 +Ethernet36 53,54,55,56 +Ethernet40 57,58,59,60 +Ethernet44 61,62,63,64 +Ethernet48 81,82,83,84 +Ethernet52 85,86,87,88 +Ethernet56 89,90,91,92 +Ethernet60 93,94,95,96 +Ethernet64 97,98,99,100 +Ethernet68 101,102,103,104 +Ethernet72 105,106,107,108 +Ethernet76 109,110,111,112 +Ethernet80 1,2,3,4 +Ethernet84 5,6,7,8 +Ethernet88 9,10,11,12 +Ethernet92 13,14,15,16 +Ethernet96 17,18,19,20 +Ethernet100 21,22,23,24 +Ethernet104 25,26,27,28 +Ethernet108 29,30,31,32 +Ethernet112 113,114,115,116 +Ethernet116 117,118,119,120 +Ethernet120 121,122,123,124 +Ethernet124 125,126,127,128 \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/ptf-port-map.ini b/.azure-pipelines/docker-sai-test-vpp/ptf-port-map.ini new file mode 100644 index 0000000000..d5143f7c99 --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/ptf-port-map.ini @@ -0,0 +1,34 @@ +# PTF port index to SAI front-panel interface mapping. +# Port names match the default sai_test resources/config_db.json (Ethernet0,4,...,124). +0@Ethernet0 +1@Ethernet4 +2@Ethernet8 +3@Ethernet12 +4@Ethernet16 +5@Ethernet20 +6@Ethernet24 +7@Ethernet28 +8@Ethernet32 +9@Ethernet36 +10@Ethernet40 +11@Ethernet44 +12@Ethernet48 +13@Ethernet52 +14@Ethernet56 +15@Ethernet60 +16@Ethernet64 +17@Ethernet68 +18@Ethernet72 +19@Ethernet76 +20@Ethernet80 +21@Ethernet84 +22@Ethernet88 +23@Ethernet92 +24@Ethernet96 +25@Ethernet100 +26@Ethernet104 +27@Ethernet108 +28@Ethernet112 +29@Ethernet116 +30@Ethernet120 +31@Ethernet124 \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/run_test.sh b/.azure-pipelines/docker-sai-test-vpp/run_test.sh new file mode 100755 index 0000000000..fe66f25dec --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/run_test.sh @@ -0,0 +1,1079 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Harness assets (e.g. vpp_startup.conf.template) live next to this script in the +# source tree and are installed under /opt/docker-sai-test-vpp in the image. When +# the image runs the script from /usr/local/bin, fall back to the install dir. +HARNESS_DIR="$SCRIPT_DIR" +[[ -f "$HARNESS_DIR/vpp_startup.conf.template" ]] || HARNESS_DIR="/opt/docker-sai-test-vpp" + +PORT_COUNT="${PORT_COUNT:-32}" +MTU="${MTU:-9100}" +# Number of PortChannel (LAG) netdevs used for cleanup at container exit. +LAG_COUNT="${LAG_COUNT:-4}" +# LAG/SVI connected-IP patterns consumed by sai_test's SIMULATE_SONIC helper +# (config/simulate_sonic.py), which assigns them in setUp when a LAG or VLAN RIF is +# created. These env vars retarget the helper's defaults to this VPP bench. +LAG_RIF_IPS="${LAG_RIF_IPS:-1}" +LAG_RIF_IPV4_PATTERN="${LAG_RIF_IPV4_PATTERN:-10.1.%d.1/24}" +LAG_RIF_IPV6_PATTERN="${LAG_RIF_IPV6_PATTERN:-fc00:1::%d:1/112}" +# VPP linux_cp host-interface (LCP tap) name prefix for a BondEthernet. +LAG_BE_TAP_PREFIX="${LAG_BE_TAP_PREFIX:-be}" + +# Assign the DUT-side connected IP for each SVI (VLAN) router interface (see +# simulate_sonic.assign_svi_rif_ips). Unlike a BondEthernet, a BVI has no linux_cp +# host-interface; the IP is set directly in VPP via vppctl once the BVI exists. +SVI_RIF_IPS="${SVI_RIF_IPS:-1}" +SVI_RIF_VLANS="${SVI_RIF_VLANS:-10:1 20:2}" +SVI_RIF_IPV4_PATTERN="${SVI_RIF_IPV4_PATTERN:-192.168.%d.1/24}" +SVI_RIF_IPV6_PATTERN="${SVI_RIF_IPV6_PATTERN:-fc02::%d:1/112}" +# VPP BVI interface name prefix for a VLAN SVI. +SVI_BVI_PREFIX="${SVI_BVI_PREFIX:-bvi}" +SAI_PROFILE="${SAI_PROFILE:-/etc/sai/sai.profile}" +SAISERVER_PORTMAP="${SAISERVER_PORTMAP:-/etc/sai/port-map.ini}" +PTF_PORTMAP="${PTF_PORTMAP:-/etc/sai/ptf-port-map.ini}" +SAI_TEST_DIR="${SAI_TEST_DIR:-/sai_test}" +TEST_RESULTS_DIR="${TEST_RESULTS_DIR:-/test-results}" +SONIC_VPP_IFMAP="${SONIC_VPP_IFMAP:-/usr/share/sonic/hwsku/sonic_vpp_ifmap.ini}" +THRIFT_PORT="${THRIFT_PORT:-9092}" +STARTUP_TIMEOUT="${STARTUP_TIMEOUT:-60}" +VPPCTL_TIMEOUT="${VPPCTL_TIMEOUT:-5}" +VPP_LOG="${VPP_LOG:-/var/log/vpp.log}" +VPP_STDOUT_LOG="${VPP_STDOUT_LOG:-/var/log/vpp-startup.log}" +VPP_API_TRACE="${VPP_API_TRACE:-/tmp/vpp-sai-api-trace.api}" +VPP_API_TRACE_TXT="${VPP_API_TRACE_TXT:-/var/log/vpp-api-trace.txt}" +SAISERVER_LOG="${SAISERVER_LOG:-/var/log/saiserver.log}" +REDIS_SOCKET="${REDIS_SOCKET:-/var/run/redis/redis.sock}" +REDIS_LOG="${REDIS_LOG:-/var/log/redis.log}" +LINKS_UP_MARKER="${LINKS_UP_MARKER:-/tmp/sai-vpp-links-up}" +LINK_UP_TRIGGER="${LINK_UP_TRIGGER:-Turn up ports...}" +KEEP_VETHS_UP_SECONDS="${KEEP_VETHS_UP_SECONDS:-120}" +KEEP_VETHS_UP_INTERVAL="${KEEP_VETHS_UP_INTERVAL:-3}" + +# Port admin-up wait tuning, consumed by the OCP test framework's +# turn_up_and_get_checked_ports() (SAI/test/sai_test/config/port_configer.py). +# In this VPP/veth harness the SAI port oper-status reads DOWN for the whole +# wait window (linux_cp carrier settles only after the config build), yet the +# links do come up and the dataplane works - so the framework's default +# per-port serial wait (32 ports x retries x interval ~= 64s) is pure dead time +# in every common-config build. We opt into the shared bounded wait (all ports +# polled together for at most retries x interval seconds) and a short interval, +# turning ~64s into a few seconds. These are exported so the ptf subprocess +# (and thus port_configer.py) sees them; unset = upstream/real-HW default. +export SAI_PORT_UP_SHARED_WAIT="${SAI_PORT_UP_SHARED_WAIT:-1}" +export SAI_PORT_UP_RETRIES="${SAI_PORT_UP_RETRIES:-2}" +export SAI_PORT_UP_POLL_INTERVAL="${SAI_PORT_UP_POLL_INTERVAL:-1}" + +# The SAI PTF T0 framework is designed to build the common switch configuration +# once and have subsequent tests reuse it (the persisted object IDs in +# /tmp/sai_model) by passing common_configured=true. When that path is NOT used, +# every test re-runs sai_create_switch + 32x sai_create_hostif inside the same +# long-lived saiserver/VPP process, which returns a null OID on the duplicate +# host-interface and crashes saiserver - so only the first test can run. With +# reuse enabled (default) the harness runs each test target as its own ptf +# invocation against one long-lived saiserver/VPP: the first builds + persists +# the common config (common_configured=false), the rest reuse it +# (common_configured=true). Set COMMON_CONFIGURED_REUSE=0 to fall back to a +# single ptf invocation for all targets (legacy behavior). +COMMON_CONFIGURED_REUSE="${COMMON_CONFIGURED_REUSE:-1}" + +# Per-test isolation. When set to 1 (default), every test target runs in its OWN +# config group: the backend (VPP + saiserver) is torn down and brought up fresh +# before each test, and each test rebuilds its own common config from scratch +# (common_configured=false) rather than reloading a `dut` persisted by a previous +# test. This eliminates all cross-test state contamination (the ECMP -6/-7 +# config-reuse artifacts and the v4/v6 NHG port-list aliasing), so the upstream OCP +# tests need no per-test workarounds. It is only affordable because the port-up +# wait was reduced from ~64s to ~6s (see SAI_PORT_UP_SHARED_WAIT and +# devdocs/progress-6-19.md); each test pays ~22s of recycle+rebuild. Set to 0 to +# fall back to config-signature grouping with persisted-config reuse (faster full +# runs, but tests then share a backend within a group). +ISOLATE_EACH_TEST="${ISOLATE_EACH_TEST:-1}" + +# Turn on sai_test's SONiC control-plane simulation (config/simulate_sonic.py): +# create PortChannel netdevs and assign LAG/SVI RIF IPs from the test setUp, since +# this standalone bench has no teamd / IntfMgr. The remaining vars retarget the +# helper's interface names / address patterns to this VPP bench. +export SIMULATE_SONIC=1 +export LAG_RIF_IPS LAG_RIF_IPV4_PATTERN LAG_RIF_IPV6_PATTERN LAG_BE_TAP_PREFIX +export SVI_RIF_IPS SVI_RIF_VLANS SVI_RIF_IPV4_PATTERN SVI_RIF_IPV6_PATTERN SVI_BVI_PREFIX +# A VLAN SVI (BVI) has no host-interface netdev, so simulate_sonic cannot use "ip". +# Provide the VPP-specific command templates it runs to program the BVI address +# directly ({ifname}/{addr} are substituted). Keeping these here (the harness layer) +# is what lets sai_test's simulate_sonic stay backend-neutral. Note: the {ifname} +# braces can't sit inside a ${VAR:-default} expansion, so guard with a plain if. +if [[ -z "${SVI_RIF_PROBE_CMD:-}" ]]; then + SVI_RIF_PROBE_CMD="timeout ${VPPCTL_TIMEOUT} vppctl show interface {ifname}" +fi +if [[ -z "${SVI_RIF_SET_IP_CMD:-}" ]]; then + SVI_RIF_SET_IP_CMD="timeout ${VPPCTL_TIMEOUT} vppctl set interface ip address {ifname} {addr}" +fi +export SVI_RIF_PROBE_CMD SVI_RIF_SET_IP_CMD +export MTU VPPCTL_TIMEOUT + +DEBUG=0 +TEST_FILTER="${TEST_FILTER:-}" +TEST_FILTERS=() +VPP_PID="" +SAISERVER_PID="" +REDIS_PID="" +VPP_CONF="" +VPP_INIT_CLI="" +KEEP_VETHS_UP_PID="" + +log() +{ + echo "[run_test] $*" +} + +die() +{ + echo "[run_test] ERROR: $*" >&2 + exit 2 +} + +usage() +{ + cat <<'EOF' +Usage: run_test.sh [--debug] [TEST_FILTER ...] + +Examples: + run_test.sh + run_test.sh sai_route_test.RouteRifTest + run_test.sh sai_route_test.RouteRifTest sai_route_test.RouteRifv6Test + run_test.sh --debug sai_route_test.RouteRifTest + +Multiple TEST_FILTERs run sequentially. By default each target runs as its own +ptf invocation against one long-lived saiserver/VPP: the first builds and +persists the common config, the rest reuse it (common_configured=true). This is +required because the VPP SAI backend cannot re-create the switch/host-interfaces +twice in one process. Set COMMON_CONFIGURED_REUSE=0 to run all targets in a +single legacy invocation. + +Without any TEST_FILTER, every discovered test class under /sai_test runs (each +as its own reuse invocation). In debug mode, VPP, saiserver, and veth +interfaces are left running for vppctl inspection. +EOF +} + +parse_args() +{ + while [[ $# -gt 0 ]]; do + case "$1" in + --debug) + DEBUG=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + break + ;; + --*) + die "unknown option: $1" + ;; + *) + break + ;; + esac + done + + if [[ $# -gt 0 ]]; then + TEST_FILTERS=("$@") + fi +} + +exec_requested_shell() +{ + if [[ $# -eq 0 ]]; then + return 0 + fi + + case "$1" in + bash|sh|/bin/bash|/bin/sh) + exec "$@" + ;; + esac +} + +require_command() +{ + local command_name="$1" + + command -v "$command_name" >/dev/null 2>&1 || die "required command not found: $command_name" +} + +require_file() +{ + local file_path="$1" + + [[ -f "$file_path" ]] || die "required file not found: $file_path" +} + +vpp_interface_name() +{ + local port_index="$1" + + echo "OEthernet${port_index}" +} + +ptf_interface_name() +{ + local port_index="$1" + + echo "OEth${port_index}_peer" +} + +preflight() +{ + [[ "${EUID}" -eq 0 ]] || die "run_test.sh must run as root; use docker run --privileged" + + require_command ip + require_command vpp + require_command vppctl + require_command saiserver + require_command ptf + require_command redis-server + require_command redis-cli + require_command timeout + + require_file "$SAI_PROFILE" + require_file "$SAISERVER_PORTMAP" + require_file "$PTF_PORTMAP" + [[ -d "$SAI_TEST_DIR" ]] || die "required directory not found: $SAI_TEST_DIR" + + mkdir -p /run/vpp /var/log "$(dirname "$REDIS_SOCKET")" "$TEST_RESULTS_DIR" "$(dirname "$SONIC_VPP_IFMAP")" +} + +start_redis() +{ + log "Starting Redis on $REDIS_SOCKET" + rm -f "$REDIS_SOCKET" + + redis-server \ + --daemonize no \ + --bind 127.0.0.1 \ + --port 0 \ + --unixsocket "$REDIS_SOCKET" \ + --unixsocketperm 777 \ + --save '' \ + --appendonly no \ + --logfile "$REDIS_LOG" & + REDIS_PID="$!" + + for ((attempt = 1; attempt <= STARTUP_TIMEOUT; attempt++)); do + if [[ -n "$REDIS_PID" ]] && ! kill -0 "$REDIS_PID" >/dev/null 2>&1; then + die "Redis exited before becoming ready; see $REDIS_LOG" + fi + + if redis-cli -s "$REDIS_SOCKET" ping >/dev/null 2>&1; then + return 0 + fi + + sleep 1 + done + + die "timed out waiting for Redis; see $REDIS_LOG" +} + +delete_veths() +{ + set +e + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + local vpp_if + local ptf_if + + vpp_if="$(vpp_interface_name "$port_index")" + ptf_if="$(ptf_interface_name "$port_index")" + + if ip link show "$vpp_if" >/dev/null 2>&1; then + ip link delete "$vpp_if" >/dev/null 2>&1 + elif ip link show "$ptf_if" >/dev/null 2>&1; then + ip link delete "$ptf_if" >/dev/null 2>&1 + fi + done + set -e +} + +disable_ipv6_autoconf() +{ + # Disable IPv6 autoconfiguration on default and all interfaces. This prevents + # the Linux kernel from automatically generating IPv6 DAD and NDP neighbor/router + # solicitation packets when interfaces are brought up. Otherwise, at scale + # (PORT_COUNT=32), VPP raw sockets are flooded with unsolicited packets causing + # level-triggered poll event starvation of CLI and binary API connections. + echo 1 > /proc/sys/net/ipv6/conf/default/disable_ipv6 2>/dev/null || true + echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 2>/dev/null || true +} + +delete_portchannels() +{ + set +e + for ((lag_index = 0; lag_index < LAG_COUNT; lag_index++)); do + local pc_if="PortChannel${lag_index}" + if ip link show "$pc_if" >/dev/null 2>&1; then + ip link delete "$pc_if" >/dev/null 2>&1 + fi + done + set -e +} + +create_veths() +{ + log "Creating ${PORT_COUNT} OEthernet/OEth peer veth pair(s)" + delete_veths + rm -f "$LINKS_UP_MARKER" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + local vpp_if + local ptf_if + + vpp_if="$(vpp_interface_name "$port_index")" + ptf_if="$(ptf_interface_name "$port_index")" + + ip link add "$vpp_if" type veth peer name "$ptf_if" + ip link set dev "$vpp_if" mtu "$MTU" + ip link set dev "$ptf_if" mtu "$MTU" + done +} + +bring_up_veths() +{ + if [[ -f "$LINKS_UP_MARKER" ]]; then + return 0 + fi + + log "Bringing up ${PORT_COUNT} OEthernet/OEth peer veth pair(s)" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + local vpp_if + local ptf_if + + vpp_if="$(vpp_interface_name "$port_index")" + ptf_if="$(ptf_interface_name "$port_index")" + + ip link set dev "$vpp_if" up + ip link set dev "$ptf_if" up + done + + : > "$LINKS_UP_MARKER" +} + +# VPP host-interfaces (host-OEthernetX) are created administratively DOWN by the +# SAI hostif-creation path. While a host-interface is admin-DOWN, linux_cp keeps +# the paired kernel netdev (OEthernetX) in NO-CARRIER/M-DOWN, which drops the +# carrier on the PTF-side veth peer (OEthX_peer) and makes the dataplane unusable +# ("Network is down" on transmit/receive). +# +# Empirically, a single `set interface state host-OEthernetX up` in VPP is enough: +# VPP comes up, linux_cp propagates the state to the kernel netdev (takes a few +# seconds to settle) and brings the peer carrier up, and it stays up afterwards +# (no flapping). Setting the state again is idempotent. So run a short-lived +# background watchdog that re-asserts admin-up for all host-interfaces across the +# host-interface-creation window; interfaces that do not exist yet are simply +# reported as errors by vppctl and ignored. +keep_vpp_veths_up() +{ + local deadline=$((SECONDS + KEEP_VETHS_UP_SECONDS)) + local vpp_up_cmds + vpp_up_cmds="$(mktemp /tmp/vpp-sai-test-keepup.XXXXXX.cmds)" + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + echo "set interface state host-OEthernet${port_index} up" >> "$vpp_up_cmds" + done + + while [[ "$SECONDS" -lt "$deadline" ]]; do + timeout "$VPPCTL_TIMEOUT" vppctl exec "$vpp_up_cmds" >/dev/null 2>&1 || true + sleep "$KEEP_VETHS_UP_INTERVAL" + done + + rm -f "$vpp_up_cmds" +} + +create_sonic_vpp_ifmap() +{ + log "Writing SONiC-to-VPP interface map to $SONIC_VPP_IFMAP" + : > "$SONIC_VPP_IFMAP" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + echo "Ethernet$((port_index * 4)) host-OEthernet${port_index}" >> "$SONIC_VPP_IFMAP" + done +} + +generate_vpp_init_cli() +{ + VPP_INIT_CLI="$(mktemp /tmp/vpp-sai-test-init.XXXXXX.cli)" + : > "$VPP_INIT_CLI" + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + echo "create host-interface name OEthernet${port_index}" >> "$VPP_INIT_CLI" + echo "set interface rx-mode host-OEthernet${port_index} interrupt" >> "$VPP_INIT_CLI" + done +} + +generate_vpp_config() +{ + VPP_CONF="$(mktemp /tmp/vpp-sai-test.XXXXXX.conf)" + local template="${HARNESS_DIR}/vpp_startup.conf.template" + local buffers_per_numa=$((PORT_COUNT * 2048)) + + sed -e "s|__VPP_LOG__|${VPP_LOG}|g" \ + -e "s|__BUFFERS_PER_NUMA__|${buffers_per_numa}|g" \ + -e "s|__VPP_INIT_CLI__|${VPP_INIT_CLI}|g" \ + "$template" > "$VPP_CONF" +} + +wait_for_vpp_ready() +{ + log "Waiting for VPP to become ready" + + for ((attempt = 1; attempt <= STARTUP_TIMEOUT; attempt++)); do + if [[ -n "$VPP_PID" ]] && ! kill -0 "$VPP_PID" >/dev/null 2>&1; then + die "VPP exited before becoming ready; see $VPP_LOG and $VPP_STDOUT_LOG" + fi + + if timeout "$VPPCTL_TIMEOUT" vppctl show version >/dev/null 2>&1; then + return 0 + fi + + sleep 1 + done + + die "timed out waiting for VPP; see $VPP_LOG and $VPP_STDOUT_LOG" +} + +verify_vpp_interfaces() +{ + log "Verifying VPP interfaces are created" + local interface_output + local port_index + local attempt + + for ((attempt = 1; attempt <= 15; attempt++)); do + local all_created=1 + interface_output="$(timeout "$VPPCTL_TIMEOUT" vppctl show interface 2>/dev/null || true)" + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + if ! grep -q "host-OEthernet${port_index}" <<< "$interface_output"; then + all_created=0 + break + fi + done + if [[ "$all_created" -eq 1 ]]; then + return 0 + fi + sleep 1 + done + + die "VPP interfaces were not created; see $VPP_LOG and $VPP_STDOUT_LOG" +} + +start_vpp() +{ + generate_vpp_init_cli + generate_vpp_config + log "Starting VPP" + vpp -c "$VPP_CONF" > "$VPP_STDOUT_LOG" 2>&1 & + VPP_PID="$!" + + wait_for_vpp_ready + verify_vpp_interfaces +} + +wait_for_saiserver_ready() +{ + log "Waiting for saiserver Thrift endpoint on 127.0.0.1:${THRIFT_PORT}" + + for ((attempt = 1; attempt <= STARTUP_TIMEOUT; attempt++)); do + if [[ -n "$SAISERVER_PID" ]] && ! kill -0 "$SAISERVER_PID" >/dev/null 2>&1; then + die "saiserver exited before becoming ready; see $SAISERVER_LOG" + fi + + if (echo >/dev/tcp/127.0.0.1/"$THRIFT_PORT") >/dev/null 2>&1; then + return 0 + fi + + sleep 1 + done + + die "timed out waiting for saiserver; see $SAISERVER_LOG" +} + +start_saiserver() +{ + log "Starting saiserver" + # libsaivs logs through SWSS_LOG_* (libswsscommon). saiserver.cpp no longer + # configures swss::Logger after the SAI standalone cleanup; LD_PRELOAD routes + # those lines to stdout so the redirect below lands in SAISERVER_LOG. + export LD_PRELOAD="/usr/local/lib/libswss_log_stdout.so${LD_PRELOAD:+:${LD_PRELOAD}}" + saiserver -p "$SAI_PROFILE" -f "$SAISERVER_PORTMAP" > "$SAISERVER_LOG" 2>&1 & + SAISERVER_PID="$!" + + wait_for_saiserver_ready +} + +ptf_target_requires_relax() +{ + case "$1" in + sai_fdb_test.BridgePortLearnDisableTest|\ + sai_fdb_test.BroadcastNoLearnTest|\ + sai_fdb_test.FdbAgingAfterMoveTest|\ + sai_fdb_test.FdbAgingTest|\ + sai_fdb_test.FdbFlushAllDynamicTest|\ + sai_fdb_test.FdbFlushAllStaticTest|\ + sai_fdb_test.FdbFlushAllTest|\ + sai_fdb_test.FdbFlushPortDynamicTest|\ + sai_fdb_test.FdbFlushPortStaticTest|\ + sai_fdb_test.FdbFlushVlanDynamicTest|\ + sai_fdb_test.FdbFlushVlanStaticTest|\ + sai_fdb_test.MulticastNoLearnTest|\ + sai_fdb_test.NonBridgePortNoLearnTest|\ + sai_fdb_test.RemoveVlanmemberLearnTest|\ + sai_fdb_test.VlanLearnDisableTest|\ + sai_route_test.SviDirectBroadcastTest|\ + sai_sanity_test.SaiSanityTest|\ + sai_tunnel_test.SviIPInIPTunnelDecapFloodV6InV4Test|\ + sai_tunnel_test.SviIPInIPTunnelDecapFloodv4Inv4Test|\ + sai_vlan_test.ArpRequestFloodingTest|\ + sai_vlan_test.BroadcastTest|\ + sai_vlan_test.DisableMacLearningTaggedTest|\ + sai_vlan_test.DisableMacLearningUntaggedTest|\ + sai_vlan_test.TaggedVlanFloodingTest|\ + sai_vlan_test.UnTaggedVlanFloodingTest) + return 0 + ;; + esac + + return 1 +} + +build_ptf_args() +{ + # Args: + # test_target a ptf test selector (module or module.Class), or empty + # for the whole /sai_test suite. + # common_configured "true" -> reuse the previously persisted common config + # "false" -> build (and persist) the common config + # "" -> do not pass the param at all (legacy) + # xunit_dir directory PTF writes its JUnit XML into. PTF rmtree's + # this dir on startup, so it must be a private per-call + # dir (NOT a bind mount, NOT shared across invocations). + local test_target="$1" + local common_configured="$2" + local xunit_dir="$3" + local test_params="thrift_server='127.0.0.1';port_map_file='$PTF_PORTMAP'" + + if [[ -n "$common_configured" ]]; then + test_params="${test_params};common_configured='${common_configured}'" + fi + + PTF_ARGS=(--test-dir "$SAI_TEST_DIR") + + for ((port_index = 0; port_index < PORT_COUNT; port_index++)); do + PTF_ARGS+=(--interface "${port_index}@$(ptf_interface_name "$port_index")") + done + + PTF_ARGS+=(--test-params "$test_params") + PTF_ARGS+=(--xunit --xunit-dir "$xunit_dir") + + # Positive flood tests expect copies on multiple ports, but PTF's flood + # verifier consumes one copy and then rejects the remaining copies via + # verify_no_other_packets(). Relax that final check only for explicitly + # classified flood targets; applying it globally would disable negative + # packet assertions in unrelated tests. + if ptf_target_requires_relax "$test_target"; then + PTF_ARGS+=(--relax) + fi + + if [[ -n "$test_target" ]]; then + PTF_ARGS+=("$test_target") + fi +} + +# Enumerate every test class under $SAI_TEST_DIR as "module.Class" selectors so +# each can run as its own config-reuse ptf invocation. Best-effort: on any +# failure prints nothing and the caller falls back to a single invocation. +enumerate_test_classes() +{ + python3 - "$SAI_TEST_DIR" <<'PY' 2>/dev/null || true +import sys, unittest +test_dir = sys.argv[1] +try: + suite = unittest.TestLoader().discover( + test_dir, pattern="sai_*_test.py", top_level_dir=test_dir) +except Exception: + sys.exit(0) +seen = [] +def walk(s): + for t in s: + if isinstance(t, unittest.TestSuite): + walk(t) + else: + cls = t.__class__ + name = "%s.%s" % (cls.__module__, cls.__name__) + if name not in seen: + seen.append(name) +walk(suite) +for n in seen: + print(n) +PY +} + +print_debug_state() +{ + set +e + log "Linux veth interfaces" + ip -br link show type veth + + if command -v vppctl >/dev/null 2>&1 && timeout "$VPPCTL_TIMEOUT" vppctl show version >/dev/null 2>&1; then + log "VPP interfaces" + timeout "$VPPCTL_TIMEOUT" vppctl show interface + log "VPP hardware interfaces" + timeout "$VPPCTL_TIMEOUT" vppctl show hardware-interfaces + log "Saving VPP API trace to $VPP_API_TRACE" + timeout "$VPPCTL_TIMEOUT" vppctl api trace save "$(basename "$VPP_API_TRACE")" + log "Writing decoded VPP API trace to $VPP_API_TRACE_TXT" + timeout "$VPPCTL_TIMEOUT" vppctl api trace dump > "$VPP_API_TRACE_TXT" 2>&1 + fi + + log "Last 200 lines of $VPP_STDOUT_LOG" + tail -n 200 "$VPP_STDOUT_LOG" 2>/dev/null + log "Last 200 lines of $VPP_LOG" + tail -n 200 "$VPP_LOG" 2>/dev/null + log "Last 200 lines of $SAISERVER_LOG" + tail -n 200 "$SAISERVER_LOG" 2>/dev/null + log "Last 200 lines of $REDIS_LOG" + tail -n 200 "$REDIS_LOG" 2>/dev/null + set -e +} + +terminate_process() +{ + local process_name="$1" + local process_pid="$2" + + if [[ -z "$process_pid" ]] || ! kill -0 "$process_pid" >/dev/null 2>&1; then + return 0 + fi + + kill "$process_pid" >/dev/null 2>&1 || true + for ((attempt = 1; attempt <= 5; attempt++)); do + if ! kill -0 "$process_pid" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + log "Force stopping $process_name" + kill -9 "$process_pid" >/dev/null 2>&1 || true +} + +# In --debug mode, hold the script (PID 1) open after the tests finish so the +# container keeps running with VPP, saiserver, Redis, and the veths still up for +# interactive `vppctl` / `docker exec` inspection. Without this the entrypoint +# would exit right after the last test and Docker would stop the container, +# tearing down exactly the state debug mode is meant to preserve. Exit the +# container with `docker rm -f ` when done. +debug_hold() +{ + [[ "$DEBUG" -eq 1 ]] || return 0 + + log "Debug mode: tests complete; holding container open for inspection." + log "Inspect with: docker exec vppctl show interface" + log "Stop with: docker rm -f " + # Sleep in a loop so the process stays in PID 1 and reaps cleanly on signal. + while true; do + sleep 3600 & + wait "$!" + done +} + +cleanup() +{ + local status="$?" + + set +e + if [[ -n "$KEEP_VETHS_UP_PID" ]]; then + kill "$KEEP_VETHS_UP_PID" >/dev/null 2>&1 || true + fi + if [[ "$status" -ne 0 || "$DEBUG" -eq 1 ]]; then + print_debug_state + fi + + if [[ "$DEBUG" -eq 1 ]]; then + log "Debug mode enabled; leaving VPP, saiserver, and veth interfaces running" + return "$status" + fi + + log "Cleaning up runtime state" + terminate_process saiserver "$SAISERVER_PID" + terminate_process vpp "$VPP_PID" + terminate_process redis "$REDIS_PID" + delete_veths + delete_portchannels + [[ -n "$VPP_CONF" ]] && rm -f "$VPP_CONF" + [[ -n "$VPP_INIT_CLI" ]] && rm -f "$VPP_INIT_CLI" + set -e + + return "$status" +} + +run_ptf() +{ + local -a targets=() + + if [[ "${#TEST_FILTERS[@]}" -gt 0 ]]; then + targets=("${TEST_FILTERS[@]}") + elif [[ -n "$TEST_FILTER" ]]; then + targets=("$TEST_FILTER") + fi + + # Legacy single invocation (reuse disabled): start the backend once and run + # exactly one ptf process for whatever was requested. + if [[ "$COMMON_CONFIGURED_REUSE" != "1" ]]; then + local legacy_target="" + [[ "${#targets[@]}" -gt 0 ]] && legacy_target="${targets[0]}" + start_backend + local legacy_rc=0 + run_one_ptf "$legacy_target" "" "legacy-0" || legacy_rc="$?" + debug_hold + exit "$legacy_rc" + fi + + # Reuse mode. Plan the run as a sequence of (group_id, target) lines, grouped + # by each test's common-config signature (the kwargs it passes to + # T0TestBase.setUp). Tests that need a DIFFERENT common config than the + # group's first test cannot reuse it (e.g. ECMP tests need next-hop groups), + # so each signature gets its own group. Every group runs against a FRESHLY + # restarted backend: the first test in the group builds + persists that + # group's config (common_configured=false), the rest reuse it + # (common_configured=true). The backend restart (fresh saiserver) is what + # makes a second full config build safe — building twice in ONE saiserver + # process crashes it (the original "one test per container" bug). + local plan + if [[ "${#targets[@]}" -gt 0 ]]; then + plan="$(plan_test_groups "${targets[@]}")" + else + log "No test target given; planning all discovered test classes" + plan="$(plan_test_groups)" + fi + + # Per-test isolation: rewrite the group id of every plan line to a unique value + # so each test gets its own freshly restarted backend and rebuilds its own + # common config (common_configured=false). The signature column is preserved for + # logging. This is what lets the upstream OCP tests stay free of config-reuse + # workarounds. + if [[ "$ISOLATE_EACH_TEST" == "1" && -n "$plan" ]]; then + plan="$(printf '%s\n' "$plan" | awk -F'\t' 'NF{printf "%d\t%s\t%s\n", NR-1, $2, $3}')" + log "ISOLATE_EACH_TEST=1: each test runs in its own group (fresh backend + own config)" + fi + + if [[ -z "$plan" ]]; then + log "Planning produced no targets; running full suite in one invocation" + start_backend + local suite_rc=0 + run_one_ptf "" "false" "suite-0" || suite_rc="$?" + debug_hold + exit "$suite_rc" + fi + + local total + total="$(printf '%s\n' "$plan" | grep -c .)" + log "Planned ${total} test target(s) across $(printf '%s\n' "$plan" | cut -f1 | sort -u | grep -c .) config group(s)" + + local overall_rc=0 + local prev_group="" + local idx_in_group=0 + local idx=0 + local group target sig common_configured rc + + while IFS=$'\t' read -r group target sig; do + [[ -z "$target" ]] && continue + idx=$((idx + 1)) + + if [[ "$group" != "$prev_group" ]]; then + # New config group: restart the backend for a clean saiserver, then + # the first test of the group rebuilds the common config. + if [[ -n "$prev_group" ]]; then + stop_backend + fi + log "### Config group ${group} (signature: ${sig:-()}) ###" + start_backend + prev_group="$group" + idx_in_group=0 + fi + + if [[ "$idx_in_group" -eq 0 ]]; then + common_configured="false" + else + common_configured="true" + fi + idx_in_group=$((idx_in_group + 1)) + + log "=== [${idx}/${total}] ${target} (group ${group}, common_configured=${common_configured}) ===" + rc=0 + run_one_ptf "$target" "$common_configured" "group-${group}-test-${idx}" || rc="$?" + if [[ "$rc" -ne 0 ]]; then + overall_rc="$rc" + log "test target '${target}' returned rc=${rc}" + fi + done <<< "$plan" + + debug_hold + exit "$overall_rc" +} + +# Start the runtime backend (Redis + VPP + saiserver + veth-up watchdog). VPP and +# saiserver are fresh processes each time this is called, so a subsequent group's +# common_configured=false config build runs in a clean saiserver and does not hit +# the duplicate-create crash. Veth netdevs persist across backend restarts; +# PortChannel netdevs are created on demand in sai_test setUp (SIMULATE_SONIC). +# across restarts; only the dataplane daemons are recycled. +start_backend() +{ + start_redis + start_vpp + start_saiserver + keep_vpp_veths_up & + KEEP_VETHS_UP_PID="$!" +} + +# Stop the runtime backend and reset per-run state so the next group starts clean. +# +# NOTE: ideally each test would restore the initial state via SAI teardown in its +# own tearDown(), avoiding this full backend recycle. That does not fully work yet +# because the VS-VPP backend's object removal is not idempotent enough to rebuild +# the whole T0 config a second time in one saiserver process: on remove, leftover +# VPP state (linux-cp pairs, bridge domains, bonds, etc.) is not always torn down, +# so the next create hits "already exists" (VPP VALUE_EXIST/-81) and fails. The +# host-interface case was fixed in sonic-sairedis PR #1952 (vs_remove_hostif now +# deletes the linux-cp pair + disables IPv6, and create tolerates VALUE_EXIST), but +# VLAN, bridge-port, LAG, RIF and route removal still need the same treatment. +# Until that is done, we restart the backend (fresh saiserver) per group so each +# config build starts from clean VPP state. See devdocs/progress-7-3-hostif-removal.md. +stop_backend() +{ + if [[ -n "$KEEP_VETHS_UP_PID" ]]; then + kill "$KEEP_VETHS_UP_PID" >/dev/null 2>&1 || true + wait "$KEEP_VETHS_UP_PID" 2>/dev/null || true + KEEP_VETHS_UP_PID="" + fi + terminate_process saiserver "$SAISERVER_PID"; SAISERVER_PID="" + terminate_process vpp "$VPP_PID"; VPP_PID="" + terminate_process redis "$REDIS_PID"; REDIS_PID="" + # Drop persisted SAI object IDs and the link-up marker so the next group's + # first test rebuilds (and re-persists) its own config and re-asserts veths. + rm -rf /tmp/sai_model 2>/dev/null || true + rm -f "$LINKS_UP_MARKER" 2>/dev/null || true +} + +# Emit a run plan: tab-separated "\t\t" lines, +# grouped by each test class's common-config signature so identical-config tests +# run together (and can reuse one config build). With no args, plans every +# discovered test class under $SAI_TEST_DIR. +plan_test_groups() +{ + SAI_TEST_DIR="$SAI_TEST_DIR" python3 - "$@" <<'PY' +import ast, os, sys, glob, collections + +test_dir = os.environ.get("SAI_TEST_DIR", "/sai_test") +targets = sys.argv[1:] + +# kwargs that do NOT change the common config (so they must not split groups) +NON_CONFIG_KW = {"skip_reason", "wait_sec"} + +mod_file = {} +for p in glob.glob(os.path.join(test_dir, "sai_*_test.py")): + mod_file[os.path.basename(p)[:-3]] = p + +# Build a cross-file class registry: name -> {bases:[...], setup:FunctionDef|None, +# module:str}. Classes can subclass intermediate test bases (e.g. EcmpBaseTestV4) +# whose setUp sets the common-config kwargs, so signatures must resolve through +# the inheritance chain, not just the class's own setUp. +registry = {} +mod_classes = collections.OrderedDict() +for mod in sorted(mod_file): + try: + tree = ast.parse(open(mod_file[mod]).read(), mod_file[mod]) + except Exception: + continue + names = [] + for node in tree.body: + if not isinstance(node, ast.ClassDef): + continue + names.append(node.name) + bases = [b.id for b in node.bases if isinstance(b, ast.Name)] + setup = next((m for m in node.body + if isinstance(m, ast.FunctionDef) and m.name == "setUp"), None) + # last definition wins if duplicated + registry[node.name] = {"bases": bases, "setup": setup, "module": mod} + mod_classes[mod] = names + +def _kwargs_from_setup_call(setup): + """Return (kwargs_str_or_None, base_to_recurse_or_None) for a class's setUp. + kwargs_str: config signature if this setUp passes config kwargs. + base_to_recurse: if setUp only chains to super()/Base.setUp() without config + kwargs, the class name to resolve the signature from.""" + for sub in ast.walk(setup): + if not (isinstance(sub, ast.Call) and isinstance(sub.func, ast.Attribute) + and sub.func.attr == "setUp"): + continue + kws = [] + for kw in sub.keywords: + if kw.arg in NON_CONFIG_KW or kw.arg is None: + continue + try: + val = ast.literal_eval(kw.value) + except Exception: + val = "?" + kws.append("%s=%s" % (kw.arg, val)) + if kws: + return ("|".join(sorted(kws)), None) + # No config kwargs: figure out which base this chains to. + f = sub.func.value + if isinstance(f, ast.Call) and isinstance(f.func, ast.Name) \ + and f.func.id == "super": + return (None, "__super__") # super().setUp() + if isinstance(f, ast.Name): + return (None, f.id) # .setUp(self, ...) + return (None, "__super__") + return ("()", None) + +def signature(clsname, seen=None): + if seen is None: + seen = set() + if clsname in seen or clsname not in registry: + return "()" # unknown base (e.g. T0TestBase) -> default config + seen.add(clsname) + info = registry[clsname] + setup = info["setup"] + if setup is None: + # inherits setUp wholesale from first base + for b in info["bases"]: + return signature(b, seen) + return "()" + sig, base = _kwargs_from_setup_call(setup) + if sig is not None: + return sig + # chained to super/base without config kwargs -> resolve that base + if base == "__super__": + for b in info["bases"]: + return signature(b, seen) + return "()" + return signature(base, seen) + +# Expand requested targets (or everything) into (module, class) pairs. +pairs = [] +if not targets: + for mod in mod_classes: + for c in mod_classes[mod]: + pairs.append((mod, c)) +else: + for t in targets: + if "." in t: + mod, cls = t.split(".", 1) + pairs.append((mod, cls)) + elif t in mod_classes: + for c in mod_classes[t]: + pairs.append((t, c)) + else: + pairs.append((t, "")) # unknown selector: run as-is, default group + +# Group by resolved signature, preserving first-seen order. +groups = collections.OrderedDict() +for mod, cls in pairs: + sig = signature(cls) if cls else "()" + groups.setdefault(sig, []).append("%s.%s" % (mod, cls) if cls else mod) + +for gid, (sig, tlist) in enumerate(groups.items()): + for t in tlist: + sys.stdout.write("%d\t%s\t%s\n" % (gid, t, sig)) +PY +} + +# Copy one PTF invocation's JUnit XML into the shared results directory without +# replacing an earlier invocation's same-named TEST-.xml report. +collect_xunit_results() +{ + local xunit_dir="$1" + local invocation_namespace="$2" + local xml_file + local xml_name + local namespaced_name + local -a xml_files + + [[ -n "$TEST_RESULTS_DIR" ]] || return 0 + + mkdir -p "$TEST_RESULTS_DIR" + shopt -s nullglob + xml_files=("$xunit_dir"/*.xml) + shopt -u nullglob + + for xml_file in "${xml_files[@]}"; do + xml_name="$(basename "$xml_file")" + if [[ "$xml_name" == TEST-* ]]; then + namespaced_name="TEST-${invocation_namespace}-${xml_name#TEST-}" + else + namespaced_name="TEST-${invocation_namespace}-${xml_name}" + fi + cp "$xml_file" "$TEST_RESULTS_DIR/$namespaced_name" + done +} + +# Run a single ptf invocation for one target. Echoes PTF output through and +# triggers the one-time veth bring-up on the framework's "Turn up ports..." +# marker. Returns ptf's exit code. +# +# PTF rmtree's its --xunit-dir on startup, so each invocation gets its own +# private temp dir (which PTF may freely wipe), and the resulting JUnit XML is +# copied into the shared $TEST_RESULTS_DIR afterward. This keeps results from +# accumulating across invocations and avoids EBUSY when $TEST_RESULTS_DIR is a +# bind mount (rmtree of a mount point fails). +run_one_ptf() +{ + local test_target="$1" + local common_configured="$2" + local invocation_namespace="$3" + local test_rc + local xunit_dir + + xunit_dir="$(mktemp -d /tmp/ptf-xunit.XXXXXX)" + + build_ptf_args "$test_target" "$common_configured" "$xunit_dir" + log "Running PTF${test_target:+ filter: $test_target}" + + set +e + PYTHONUNBUFFERED=1 ptf "${PTF_ARGS[@]}" 2>&1 | while IFS= read -r ptf_line; do + printf '%s\n' "$ptf_line" + + case "$ptf_line" in + *"Turn up ports..."*) + bring_up_veths + ;; + esac + done + test_rc="${PIPESTATUS[0]}" + set -e + + collect_xunit_results "$xunit_dir" "$invocation_namespace" + rm -rf "$xunit_dir" + + return "$test_rc" +} + +main() +{ + exec_requested_shell "$@" + parse_args "$@" + trap cleanup EXIT + + preflight + disable_ipv6_autoconf + create_veths + create_sonic_vpp_ifmap + run_ptf +} + +main "$@" \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/sai.profile b/.azure-pipelines/docker-sai-test-vpp/sai.profile new file mode 100644 index 0000000000..d8de1b9d5b --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/sai.profile @@ -0,0 +1,3 @@ +SAI_VS_SWITCH_TYPE=SAI_VS_SWITCH_TYPE_VPP +SAI_VS_HOSTIF_USE_TAP_DEVICE=true +SAI_VS_INTERFACE_LANE_MAP_FILE=/etc/sai/lanemap.ini \ No newline at end of file diff --git a/.azure-pipelines/docker-sai-test-vpp/swss_log_stdout_preload.cpp b/.azure-pipelines/docker-sai-test-vpp/swss_log_stdout_preload.cpp new file mode 100644 index 0000000000..53c479911b --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/swss_log_stdout_preload.cpp @@ -0,0 +1,16 @@ +// Harness-only LD_PRELOAD shim: route SAI VS library SWSS_LOG_* to stdout so +// run_test.sh capture in /var/log/sai-server.log works without pulling SONiC +// logger setup back into the SAI server binary. +#include + +namespace { + +__attribute__((constructor)) +static void route_swss_log_to_stdout() +{ + SWSS_LOG_ENTER(); + swss::Logger::setMinPrio(swss::Logger::SWSS_DEBUG); + swss::Logger::swssOutputNotify("saiserver", "STDOUT"); +} + +} // namespace diff --git a/.azure-pipelines/docker-sai-test-vpp/vpp_startup.conf.template b/.azure-pipelines/docker-sai-test-vpp/vpp_startup.conf.template new file mode 100644 index 0000000000..45c44ff07e --- /dev/null +++ b/.azure-pipelines/docker-sai-test-vpp/vpp_startup.conf.template @@ -0,0 +1,55 @@ +unix { + nodaemon + log __VPP_LOG__ + full-coredump + cli-listen /run/vpp/cli.sock + poll-sleep-usec 100 + exec __VPP_INIT_CLI__ +} + +api-trace { + on + nitems 32768 +} + +api-segment { + global-size 256M + api-size 64M +} + +socksvr { + default +} + +memory { + main-heap-size 4G +} + +l3fib { + fib-entry-pool-size 256K + load-balance-pool-size 256K + ip4-mtrie-pool-size 256K +} + +ip6 { + heap-size 128M +} + +plugins { + plugin default { disable } + plugin af_packet_plugin.so { enable } + plugin linux_cp_plugin.so { enable } + plugin linux_nl_plugin.so { enable } + plugin acl_plugin.so { enable } + plugin vxlan_plugin.so { enable } + plugin tunterm_acl_plugin.so { enable } + plugin ip_validate_plugin.so { enable } +} + +linux-cp { + lcp-auto-subint +} + +buffers { + buffers-per-numa __BUFFERS_PER_NUMA__ +} diff --git a/.gitignore b/.gitignore index db4a306321..133898d615 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ debian/autoreconf.after debian/autoreconf.before debian/debhelper-build-stamp debian/files +.azure-pipelines/docker-sai-test-vpp/debs/ debian/python-pysairedis/ debian/python3-pysairedis/