From 109644a4c1898979fb3250835e4c8abfa4a048ec Mon Sep 17 00:00:00 2001 From: Ximon Eighteen <3304436+ximon18@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:56:21 +0200 Subject: [PATCH] Automated code coverage reporting for system tests. Adds +coverage which: - Creates a run specific tmp dir in /tmp/cascade-coverage-data/xxx. - Instruments cascade and cascaded using RUSTFLAGS="-C instrument-coverage". (see https://doc.rust-lang.org/beta/rustc/instrument-coverage.html) - Sets LLVM_PROFILE_FILE to control where instrumentation counter data is written to. Uses uuid and GITHUB_JOB to write data to unique paths per matrix and job run. Exports LLVM_PROFILE_FILE to the env of the job so that all invocations of cascade and cascaded pick it up. - Adds pkill cascaded to all system test jobs to ensure that coverage data gets written out before the test container is destroyed. - Generates a HTML report in /tmp/cascade-coverage-data/xxx/report. --- act-wrapper | 61 ++++++++++++++++++- .../setup-and-start-cascade/action.yml | 13 ++++ .../cascade-test-image/Dockerfile | 3 + .../tests/add-zone-query/action.yml | 8 +++ .../tests/all-rr-types/action.yml | 3 + .../tests/downstream-tsig/action.yml | 3 + .../incremental-signing-nsec-nsec3/action.yml | 3 + .../tests/incremental-signing/action.yml | 3 + integration-tests/tests/ixfr-in/action.yml | 3 + .../tests/load-signed-apex-cname/action.yml | 3 + .../tests/load-zonefile/action.yml | 9 +++ .../tests/multipart-axfr/action.yml | 3 + .../notify-in-for-unknown-zone/action.yml | 3 + .../tests/persist-zone/action.yml | 33 +++++----- .../tests/remove-zone/action.yml | 3 + .../tests/review-signed-zone/action.yml | 3 + .../tests/review-unsigned-zone/action.yml | 7 +++ .../tests/review-unsigned-zone2/action.yml | 3 + .../tests/sign-using-a-hsm/action.yml | 3 + .../tests/strip-dnssec-rrs/action.yml | 3 + .../tests/test-template/action.yml | 5 ++ .../tests/upstream-tsig/action.yml | 3 + 22 files changed, 164 insertions(+), 17 deletions(-) diff --git a/act-wrapper b/act-wrapper index e74275523..88d409823 100755 --- a/act-wrapper +++ b/act-wrapper @@ -29,6 +29,10 @@ function usage { echo echo "'$0 +stats-graph '" echo + echo "To create a code coverage report use:" + echo + echo "'$0 +coverage '" + echo echo "NOTE: '+XXX' arguments must be the first arguments to act-wrapper, and can be combined." echo echo "Inputs that can be provided via --input:" @@ -84,6 +88,13 @@ while [[ "$1" =~ ^\+ ]]; do shift docker_stats_graph_path="$1" ;; + +coverage) + coverage=true + coverage_path="/tmp/cascade-coverage-data/$BASHPID" + mkdir -p ${coverage_path} + act_args+=(--input build-profile=debug) + act_args+=(--container-options "-v ${coverage_path}:/tmp/cov") + ;; *) echo >&2 "ERROR: Unrecognized + argument: $1" usage @@ -93,10 +104,38 @@ while [[ "$1" =~ ^\+ ]]; do shift done +if [[ "$coverage" == true ]]; then + if [[ "$no_build" == true || "$build_inside" == true ]]; then + echo >&2 "ERROR: +coverage is not compatible with +nobuild or +build-inside" + exit 1 + fi + + echo "Code coverage reporting activated:" + echo " - Data will be written to the following directory: ${coverage_path}" +fi + if [[ "$no_build" != true ]]; then if [[ "$build_inside" != true ]]; then - cargo build - cargo build --release + if [[ "$coverage" == true ]]; then + # Ensure that only the binaries we want to be instrumented get + # instrumented, so don't export RUSTFLAGS but instead set it specifically + # here. See: https://doc.rust-lang.org/beta/rustc/instrument-coverage.html + # for more information. + RUSTFLAGS="-C instrument-coverage" cargo build + RUSTFLAGS="-C instrument-coverage" cargo build --release + else + cargo build + cargo build --release + fi + + if [[ "$coverage" == "true" ]]; then + # For some reason I don't understand, RUSTFLAGS="-C instrument-coverage" + # causes default_xxx.profraw files to be written out _during cargo build_ + # while we only want and expect them when the built binary is run. Delete + # the files that were just created to prevent confusion and littering the + # users checkout with these files... + find . -type f -name 'default*.profraw' -exec rm {} \; + fi fi docker buildx build . -f integration-tests/cascade-test-image/Dockerfile -t nlnetlabs/cascade-tests-runner --build-arg MSRV=$MSRV "${args[@]}" @@ -223,6 +262,24 @@ EOF fi fi +if [[ "$coverage" == true ]]; then + echo "Merging coverage data files..." + llvm-profdata merge -sparse $(find "${coverage_path}/" -type f -iname '*.profraw' -printf "%p ") -o "${coverage_path}/merged" + + echo "Generating HTML coverage report in ${coverage_path}/report" + llvm-cov show \ + -Xdemangler=rustfilt \ + --object target/debug/cascade \ + --object target/debug/cascaded \ + -show-line-counts-or-regions \ + -show-instantiations \ + -ignore-filename-regex='/.cargo/' \ + -ignore-filename-regex='/.rustup/' \ + -instr-profile="${coverage_path}/merged" \ + --format=html \ + -output-dir="${coverage_path}/report" +fi + if docker container ls --format '{{.Names}}' --all | grep -q "^act-"; then echo "NOTE: act seemingly failed to clean up all containers it created." echo "You may wish to check and clean up lingering containers" diff --git a/integration-tests/.github/actions/setup-and-start-cascade/action.yml b/integration-tests/.github/actions/setup-and-start-cascade/action.yml index 19f38c6f0..115f376d0 100644 --- a/integration-tests/.github/actions/setup-and-start-cascade/action.yml +++ b/integration-tests/.github/actions/setup-and-start-cascade/action.yml @@ -39,6 +39,19 @@ runs: export CASCADE_FAKETIME=${{ inputs.fake-time }} fi + # If code coverage generation is enabled, configure where the + # instrumented binaries will write coverage counter metrics to. The path + # /tmp/cov is expected to be mounted from the host into the container + # in order to be able to access the counters once the test has completed + # and to aggregate them across multiple test jobs. However this means + # that we need to make sure that the files we write to have unique paths + # per job invocation, even different matrix invocations of the same job, + # so that they don't overwrite each others data. + export LLVM_PROFILE_FILE="/tmp/cov/${GITHUB_JOB}/$(uuid)/cascade-%p-%h-%m.profraw" + + # Does this work? + echo "LLVM_PROFILE_FILE=${LLVM_PROFILE_FILE}" >> "$GITHUB_ENV" + mkdir -p "${CASCADE_DIR}/policies" # We need to create a custom config for cascade because the default paths are not available/permitted in this environment cascade template config | \ diff --git a/integration-tests/cascade-test-image/Dockerfile b/integration-tests/cascade-test-image/Dockerfile index c6fb441bb..d42c30e46 100644 --- a/integration-tests/cascade-test-image/Dockerfile +++ b/integration-tests/cascade-test-image/Dockerfile @@ -31,6 +31,9 @@ RUN < dig-query.log + - name: Tell the NSD secondary that the zone is now available # Only necessary if NOTIFY is not setup for cascade run: | ./integration-tests/scripts/manage-test-environment.sh control nsd-secondary transfer example.test + - name: The new zone should now be available via the resolver run: | dig text.example.test TXT | grep "Hello World" + + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/all-rr-types/action.yml b/integration-tests/tests/all-rr-types/action.yml index 2f33be11b..542d59dc1 100644 --- a/integration-tests/tests/all-rr-types/action.yml +++ b/integration-tests/tests/all-rr-types/action.yml @@ -43,6 +43,9 @@ runs: sleep 1 done + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/downstream-tsig/action.yml b/integration-tests/tests/downstream-tsig/action.yml index 30c576f87..ce830870e 100644 --- a/integration-tests/tests/downstream-tsig/action.yml +++ b/integration-tests/tests/downstream-tsig/action.yml @@ -155,6 +155,9 @@ runs: run: | dig @127.0.0.1 -p 1054 text.${{ steps.set-vars.outputs.ZONE_NAME }} TXT | grep 'Hello World' + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/incremental-signing-nsec-nsec3/action.yml b/integration-tests/tests/incremental-signing-nsec-nsec3/action.yml index 13685e452..60c294694 100644 --- a/integration-tests/tests/incremental-signing-nsec-nsec3/action.yml +++ b/integration-tests/tests/incremental-signing-nsec-nsec3/action.yml @@ -138,6 +138,9 @@ runs: dig @127.0.0.1 -p 4542 example AXFR | egrep -v '^;|^$' | sort -u > output.sorted diff -w -u output.sorted reference.output.sorted + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/incremental-signing/action.yml b/integration-tests/tests/incremental-signing/action.yml index a8397b90b..cd7e4e2d2 100644 --- a/integration-tests/tests/incremental-signing/action.yml +++ b/integration-tests/tests/incremental-signing/action.yml @@ -116,6 +116,9 @@ runs: dig @127.0.0.1 -p 4542 example AXFR | egrep -v '^;|^$' | sort -u > output.sorted diff -w -u output.sorted reference.output.sorted + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/ixfr-in/action.yml b/integration-tests/tests/ixfr-in/action.yml index ac1484497..1a4545e43 100644 --- a/integration-tests/tests/ixfr-in/action.yml +++ b/integration-tests/tests/ixfr-in/action.yml @@ -204,6 +204,9 @@ runs: grep -qF "www.example.test.,5,IN,A,192.168.0.1" dig-query.log grep -qF "mail.example.test.,5,IN,MX,20,example.test." dig-query.log + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/load-signed-apex-cname/action.yml b/integration-tests/tests/load-signed-apex-cname/action.yml index 2d41beca1..82d30f48e 100644 --- a/integration-tests/tests/load-signed-apex-cname/action.yml +++ b/integration-tests/tests/load-signed-apex-cname/action.yml @@ -45,6 +45,9 @@ runs: sleep 1 done + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/load-zonefile/action.yml b/integration-tests/tests/load-zonefile/action.yml index 37de91efc..a13819c89 100644 --- a/integration-tests/tests/load-zonefile/action.yml +++ b/integration-tests/tests/load-zonefile/action.yml @@ -46,9 +46,11 @@ runs: mail MX 10 example.test. text TXT "Hello World!" EOF + - name: Add a zone run: | cascade zone add --policy default --source $PWD/example.test.zone example.test + - name: Check zone status run: | timeout=10 # seconds @@ -61,16 +63,23 @@ runs: fi sleep 1 done + - name: Query zone run: | dig @127.0.0.1 -p 4542 example.test AXFR + - name: Tell NSD that the zone is now available # Only necessary if NOTIFY is not setup for cascade run: | ./integration-tests/scripts/manage-test-environment.sh control nsd-secondary transfer example.test + - name: The new zone should now be available via the resolver run: | dig text.example.test TXT | grep "Hello World" + + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/multipart-axfr/action.yml b/integration-tests/tests/multipart-axfr/action.yml index 0ac045a8d..d0eed224b 100644 --- a/integration-tests/tests/multipart-axfr/action.yml +++ b/integration-tests/tests/multipart-axfr/action.yml @@ -103,6 +103,9 @@ runs: exit 1 fi + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/notify-in-for-unknown-zone/action.yml b/integration-tests/tests/notify-in-for-unknown-zone/action.yml index 7ea7fdd02..e2d30372f 100644 --- a/integration-tests/tests/notify-in-for-unknown-zone/action.yml +++ b/integration-tests/tests/notify-in-for-unknown-zone/action.yml @@ -39,6 +39,9 @@ runs: sleep 4 cascade health + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/persist-zone/action.yml b/integration-tests/tests/persist-zone/action.yml index 36cc80614..03edf0425 100644 --- a/integration-tests/tests/persist-zone/action.yml +++ b/integration-tests/tests/persist-zone/action.yml @@ -56,9 +56,9 @@ runs: done - name: Restart Cascade - run: | - CASCADE_DIR=${{ github.workspace }}/cascade-dir - cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + uses: ./.github/actions/setup-and-start-cascade + with: + log-level: ${{ inputs.log-level }} - name: Wait for Cascade to become healthy run: | @@ -152,9 +152,9 @@ runs: # Cascade should reload the signed zone data from the data it persisted # to disk. - name: Start Cascade again - run: | - CASCADE_DIR=${{ github.workspace }}/cascade-dir - cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + uses: ./.github/actions/setup-and-start-cascade + with: + log-level: ${{ inputs.log-level }} - name: Wait for Cascade to become healthy again run: | @@ -232,9 +232,9 @@ runs: rm "${CASCADE_DIR}"/zone-state/example.test.signed.* - name: Start Cascade a 3rd time - run: | - CASCADE_DIR=${{ github.workspace }}/cascade-dir - cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + uses: ./.github/actions/setup-and-start-cascade + with: + log-level: ${{ inputs.log-level }} - name: Wait for Cascade to become healthy a 3rd time run: | @@ -498,9 +498,9 @@ runs: done - name: Start Cascade a 4th time - run: | - CASCADE_DIR=${{ github.workspace }}/cascade-dir - cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + uses: ./.github/actions/setup-and-start-cascade + with: + log-level: ${{ inputs.log-level }} - name: Wait for Cascade to become healthy a 4th time run: | @@ -625,9 +625,9 @@ runs: rm "${CASCADE_DIR}"/zone-state/example.test.signed.* - name: Start Cascade again - run: | - CASCADE_DIR=${{ github.workspace }}/cascade-dir - cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + uses: ./.github/actions/setup-and-start-cascade + with: + log-level: ${{ inputs.log-level }} - name: Wait for Cascade to become healthy again run: | @@ -682,6 +682,9 @@ runs: exit 1 fi + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/remove-zone/action.yml b/integration-tests/tests/remove-zone/action.yml index 59f7bde50..874bd37bb 100644 --- a/integration-tests/tests/remove-zone/action.yml +++ b/integration-tests/tests/remove-zone/action.yml @@ -85,6 +85,9 @@ runs: - name: Check that Cascade is still running run: cascade health + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/review-signed-zone/action.yml b/integration-tests/tests/review-signed-zone/action.yml index 3b58f83ed..a303d613b 100644 --- a/integration-tests/tests/review-signed-zone/action.yml +++ b/integration-tests/tests/review-signed-zone/action.yml @@ -118,6 +118,9 @@ runs: run: | dig @127.0.0.1 -p 4542 example.test SOA | grep -F 'status: NOERROR' + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/review-unsigned-zone/action.yml b/integration-tests/tests/review-unsigned-zone/action.yml index 94d3f48f1..8a3569579 100644 --- a/integration-tests/tests/review-unsigned-zone/action.yml +++ b/integration-tests/tests/review-unsigned-zone/action.yml @@ -38,9 +38,11 @@ runs: chmod +x ${INTEGRATION_TEST_DIR}/hooks/* # Tell Cascade to load our new test policy. cascade policy reload + - name: Add a zone run: | cascade zone add --policy test --source 127.0.0.1:1055 example.test + - name: Check zone status run: | timeout=10 # seconds @@ -53,9 +55,14 @@ runs: fi sleep 1 done + - name: Query zone run: | dig @127.0.0.1 -p 4542 example.test AXFR + + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/review-unsigned-zone2/action.yml b/integration-tests/tests/review-unsigned-zone2/action.yml index b1ac189bd..b9143beb1 100644 --- a/integration-tests/tests/review-unsigned-zone2/action.yml +++ b/integration-tests/tests/review-unsigned-zone2/action.yml @@ -81,6 +81,9 @@ runs: sleep 1 done + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/sign-using-a-hsm/action.yml b/integration-tests/tests/sign-using-a-hsm/action.yml index f48e73bba..729527eeb 100644 --- a/integration-tests/tests/sign-using-a-hsm/action.yml +++ b/integration-tests/tests/sign-using-a-hsm/action.yml @@ -64,6 +64,9 @@ runs: run: | cascade zone status --detailed example.test | grep -Fq 'kmip://' + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/strip-dnssec-rrs/action.yml b/integration-tests/tests/strip-dnssec-rrs/action.yml index 5d0cd5195..de78606a0 100644 --- a/integration-tests/tests/strip-dnssec-rrs/action.yml +++ b/integration-tests/tests/strip-dnssec-rrs/action.yml @@ -49,6 +49,9 @@ runs: dig +noall +answer +onesoa @127.0.0.1 -p 4542 example.test AXFR > example-test.axfr.log dnssec-verify -o example.test example-test.axfr.log | tee dnssec-verify.log + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/test-template/action.yml b/integration-tests/tests/test-template/action.yml index 02ab32229..59736bdc2 100644 --- a/integration-tests/tests/test-template/action.yml +++ b/integration-tests/tests/test-template/action.yml @@ -25,9 +25,14 @@ runs: - uses: ./.github/actions/setup-and-start-cascade with: log-level: ${{ inputs.log-level }} + ### ### Create your test here ### + + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure() diff --git a/integration-tests/tests/upstream-tsig/action.yml b/integration-tests/tests/upstream-tsig/action.yml index 2e16a74a5..0637f4e05 100644 --- a/integration-tests/tests/upstream-tsig/action.yml +++ b/integration-tests/tests/upstream-tsig/action.yml @@ -113,6 +113,9 @@ runs: NUM_LINES=$(cascade tsig list | wc -l) [[ ${NUM_LINES} -eq 0 ]] || exit 1 + - name: Kill Cascade so that coverage metrics, if enabled, get written out. + run: pkill cascaded + - name: Print log files on any failure in this job uses: ./.github/actions/print-logfiles if: failure()