Automated code coverage reporting for system tests.

act-wrapper

@@ -42,6 +42,10 @@ function usage {
 	echo "  If +stats-report is not used, a temporary file will be used"
 	echo "  to capture resource usage in CSV format."
 	echo
+	echo "To create a code coverage report use:"
+	echo
+	echo "'$0 +coverage </path/to/write/report/>'"
+	echo
 	echo "NOTE: '+XXX' arguments must be the first arguments to act-wrapper, and can be combined."
 	echo
 	echo "Inputs that can be provided via --input:"
@@ -97,6 +101,13 @@ while [[ "$1" =~ ^\+ ]]; do
 			shift
 			docker_stats_graph_path="$1"
 			;;
+		+coverage)
+		  coverage=true
+		  coverage_path="/tmp/cascade-coverage-data/$BASHPID"
+		  mkdir -p ${coverage_path}
+		  act_args+=(--input build-profile=debug)
+		  act_args+=(--container-options "-v ${coverage_path}:/tmp/cov")
+		  ;;
 		+benchmark)
 		  if [ $# -ne 5 ]; then
 		  	echo >&2 "ERROR: Incorrect number of arguments for +benchmark."
@@ -128,10 +139,38 @@ while [[ "$1" =~ ^\+ ]]; do
 	shift
 done
 
+if [[ "$coverage" == true ]]; then
+	if [[ "$no_build" == true || "$build_inside" == true ]]; then
+  	echo >&2 "ERROR: +coverage is not compatible with +nobuild or +build-inside"
+	  exit 1
+	fi
+
+  echo "Code coverage reporting activated:"
+  echo "  - Data will be written to the following directory: ${coverage_path}"
+fi
+
 if [[ "$no_build" != true ]]; then
 	if [[ "$build_inside" != true ]]; then
-		cargo build
-		cargo build --release
+		if [[ "$coverage" == true ]]; then
+			# Ensure that only the binaries we want to be instrumented get
+			# instrumented, so don't export RUSTFLAGS but instead set it specifically
+			# here. See: https://doc.rust-lang.org/beta/rustc/instrument-coverage.html
+			# for more information.
+			RUSTFLAGS="-C instrument-coverage" cargo build
+			RUSTFLAGS="-C instrument-coverage" cargo build --release
+		else
+			cargo build
+			cargo build --release
+		fi
+
+		if [[ "$coverage" == "true" ]]; then
+			# For some reason I don't understand, RUSTFLAGS="-C instrument-coverage"
+			# causes default_xxx.profraw files to be written out _during cargo build_
+			# while we only want and expect them when the built binary is run. Delete
+			# the files that were just created to prevent confusion and littering the
+			# users checkout with these files...
+			find . -type f -name 'default*.profraw' -exec rm {} \;
+		fi
 	fi
 
 	docker buildx build . -f integration-tests/cascade-test-image/Dockerfile -t nlnetlabs/cascade-tests-runner --build-arg MSRV=$MSRV "${args[@]}"
@@ -258,6 +297,24 @@ EOF
 	fi
 fi
 
+if [[ "$coverage" == true ]]; then
+	echo "Merging coverage data files..."
+	llvm-profdata merge -sparse $(find "${coverage_path}/" -type f -iname '*.profraw' -printf "%p ") -o "${coverage_path}/merged"
+
+	echo "Generating HTML coverage report in ${coverage_path}/report"
+	llvm-cov show \
+    -Xdemangler=rustfilt \
+    --object target/debug/cascade \
+    --object target/debug/cascaded \
+    -show-line-counts-or-regions \
+    -show-instantiations \
+    -ignore-filename-regex='/.cargo/' \
+    -ignore-filename-regex='/.rustup/' \
+    -instr-profile="${coverage_path}/merged" \
+    --format=html \
+    -output-dir="${coverage_path}/report"
+fi
+
 if docker container ls --format '{{.Names}}' --all | grep -q "^act-"; then
 	echo "NOTE: act seemingly failed to clean up all containers it created."
 	echo "You may wish to check and clean up lingering containers"

integration-tests/.github/actions/setup-and-start-cascade/action.yml

@@ -39,6 +39,19 @@ runs:
         export CASCADE_FAKETIME=${{ inputs.fake-time }}
       fi
 
+      # If code coverage generation is enabled, configure where the
+      # instrumented binaries will write coverage counter metrics to. The path
+      # /tmp/cov is expected to be mounted from the host into the container
+      # in order to be able to access the counters once the test has completed
+      # and to aggregate them across multiple test jobs. However this means
+      # that we need to make sure that the files we write to have unique paths
+      # per job invocation, even different matrix invocations of the same job,
+      # so that they don't overwrite each others data.
+      export LLVM_PROFILE_FILE="/tmp/cov/${GITHUB_JOB}/$(uuid)/cascade-%p-%h-%m.profraw"
+
+      # Does this work?
+      echo "LLVM_PROFILE_FILE=${LLVM_PROFILE_FILE}" >> "$GITHUB_ENV"
+
       mkdir -p "${CASCADE_DIR}/policies"
       # We need to create a custom config for cascade because the default paths are not available/permitted in this environment
       cascade template config | \

integration-tests/cascade-test-image/Dockerfile

@@ -31,6 +31,9 @@ RUN <<EOF
   apt-get install -y file
 
   apt-get install -y ldnsutils
+
+  # For use by code coverage testing
+  apt-get install -y uuid
 EOF
 
 RUN <<EOF

integration-tests/tests/add-zone-query/action.yml

@@ -29,6 +29,7 @@ runs:
     - name: Add a zone served by the NSD primary
       run: |
         cascade zone add --policy default --source 127.0.0.1:1055 example.test
+
     - name: Check zone status
       run: |
         timeout=10 # seconds
@@ -41,16 +42,23 @@ runs:
           fi
           sleep 1
         done
+
     - name: Query zone
       run: |
         dig @127.0.0.1 -p 4542 example.test AXFR > dig-query.log
+
     - name: Tell the NSD secondary that the zone is now available
       # Only necessary if NOTIFY is not setup for cascade
       run: |
         ./integration-tests/scripts/manage-test-environment.sh control nsd-secondary transfer example.test
+
     - name: The new zone should now be available via the resolver
       run: |
         dig text.example.test TXT | grep "Hello World"
+
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/all-rr-types/action.yml

@@ -43,6 +43,9 @@ runs:
           sleep 1
         done
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/downstream-tsig/action.yml

@@ -155,6 +155,9 @@ runs:
       run: |
         dig @127.0.0.1 -p 1054 text.${{ steps.set-vars.outputs.ZONE_NAME }} TXT | grep 'Hello World'
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/incremental-signing-nsec-nsec3/action.yml

@@ -138,6 +138,9 @@ runs:
         dig @127.0.0.1 -p 4542 example AXFR | egrep -v '^;|^$' | sort -u > output.sorted
         diff -w -u output.sorted reference.output.sorted
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/incremental-signing/action.yml

@@ -116,6 +116,9 @@ runs:
         dig @127.0.0.1 -p 4542 example AXFR | egrep -v '^;|^$' | sort -u > output.sorted
         diff -w -u output.sorted reference.output.sorted
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/ixfr-in/action.yml

@@ -204,6 +204,9 @@ runs:
         grep -qF "www.example.test.,5,IN,A,192.168.0.1" dig-query.log
         grep -qF "mail.example.test.,5,IN,MX,20,example.test." dig-query.log
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/load-signed-apex-cname/action.yml

@@ -45,6 +45,9 @@ runs:
           sleep 1
         done
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/load-zonefile/action.yml

@@ -46,9 +46,11 @@ runs:
         mail        MX  10 example.test.
         text        TXT "Hello World!"
         EOF
+
     - name: Add a zone
       run: |
         cascade zone add --policy default --source $PWD/example.test.zone example.test
+
     - name: Check zone status
       run: |
         timeout=10 # seconds
@@ -61,16 +63,23 @@ runs:
           fi
           sleep 1
         done
+
     - name: Query zone
       run: |
         dig @127.0.0.1 -p 4542 example.test AXFR
+
     - name: Tell NSD that the zone is now available
       # Only necessary if NOTIFY is not setup for cascade
       run: |
         ./integration-tests/scripts/manage-test-environment.sh control nsd-secondary transfer example.test
+
     - name: The new zone should now be available via the resolver
       run: |
         dig text.example.test TXT | grep "Hello World"
+
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/multipart-axfr/action.yml

@@ -100,6 +100,9 @@ runs:
           exit 1
         fi
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/notify-in-for-unknown-zone/action.yml

@@ -39,6 +39,9 @@ runs:
         sleep 4
         cascade health
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/persist-zone/action.yml

@@ -56,9 +56,9 @@ runs:
         done
 
     - name: Restart Cascade
-      run: |
-        CASCADE_DIR=${{ github.workspace }}/cascade-dir
-        cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" 
+      uses: ./.github/actions/setup-and-start-cascade
+      with:
+        log-level: ${{ inputs.log-level }}
 
     - name: Wait for Cascade to become healthy
       run: |
@@ -152,9 +152,9 @@ runs:
     # Cascade should reload the signed zone data from the data it persisted
     # to disk.
     - name: Start Cascade again
-      run: |
-        CASCADE_DIR=${{ github.workspace }}/cascade-dir
-        cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log"      
+      uses: ./.github/actions/setup-and-start-cascade
+      with:
+        log-level: ${{ inputs.log-level }}
 
     - name: Wait for Cascade to become healthy again
       run: |
@@ -232,9 +232,9 @@ runs:
         rm "${CASCADE_DIR}"/zone-state/example.test.signed.*
 
     - name: Start Cascade a 3rd time
-      run: |
-        CASCADE_DIR=${{ github.workspace }}/cascade-dir
-        cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log"      
+      uses: ./.github/actions/setup-and-start-cascade
+      with:
+        log-level: ${{ inputs.log-level }}
 
     - name: Wait for Cascade to become healthy a 3rd time
       run: |
@@ -498,9 +498,9 @@ runs:
         done
 
     - name: Start Cascade a 4th time
-      run: |
-        CASCADE_DIR=${{ github.workspace }}/cascade-dir
-        cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log"      
+      uses: ./.github/actions/setup-and-start-cascade
+      with:
+        log-level: ${{ inputs.log-level }}
 
     - name: Wait for Cascade to become healthy a 4th time
       run: |
@@ -625,9 +625,9 @@ runs:
         rm "${CASCADE_DIR}"/zone-state/example.test.signed.*
 
     - name: Start Cascade again
-      run: |
-        CASCADE_DIR=${{ github.workspace }}/cascade-dir
-        cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log"      
+      uses: ./.github/actions/setup-and-start-cascade
+      with:
+        log-level: ${{ inputs.log-level }}
 
     - name: Wait for Cascade to become healthy again
       run: |
@@ -682,6 +682,9 @@ runs:
           exit 1
         fi
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/remove-zone/action.yml

@@ -85,6 +85,9 @@ runs:
     - name: Check that Cascade is still running
       run: cascade health
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/review-signed-zone/action.yml

@@ -118,6 +118,9 @@ runs:
       run: |
         dig @127.0.0.1 -p 4542 example.test SOA | grep -F 'status: NOERROR'
 
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()

integration-tests/tests/review-unsigned-zone/action.yml

@@ -38,9 +38,11 @@ runs:
         chmod +x ${INTEGRATION_TEST_DIR}/hooks/*
         # Tell Cascade to load our new test policy.
         cascade policy reload
+
     - name: Add a zone
       run: |
         cascade zone add --policy test --source 127.0.0.1:1055 example.test
+
     - name: Check zone status
       run: |
         timeout=10 # seconds
@@ -53,9 +55,14 @@ runs:
           fi
           sleep 1
         done
+
     - name: Query zone
       run: |
         dig @127.0.0.1 -p 4542 example.test AXFR 
+
+    - name: Kill Cascade so that coverage metrics, if enabled, get written out.
+      run: pkill cascaded
+
     - name: Print log files on any failure in this job
       uses: ./.github/actions/print-logfiles
       if: failure()