From fd38d4edd070d5d10de2c6f050ff12d269a1196e Mon Sep 17 00:00:00 2001 From: nhanasi Date: Wed, 15 Jul 2026 15:27:28 -0400 Subject: [PATCH 01/25] Create L1_L2_CoverageReport --- .github/workflows/L1_L2_CoverageReport | 447 +++++++++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 .github/workflows/L1_L2_CoverageReport diff --git a/.github/workflows/L1_L2_CoverageReport b/.github/workflows/L1_L2_CoverageReport new file mode 100644 index 00000000..6ec60739 --- /dev/null +++ b/.github/workflows/L1_L2_CoverageReport @@ -0,0 +1,447 @@ +name: L1 and L2 Coverage Report + +on: + pull_request: + branches: [ develop, main ] + +permissions: + contents: write + pull-requests: write + +jobs: + l1-coverage: + name: L1 unit coverage metrics + runs-on: ubuntu-latest + container: + image: ghcr.io/rdkcentral/docker-rdk-ci:latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run unit tests with coverage enabled + run: sh run_ut.sh --enable-cov + + - name: Extract line and function coverage + id: cov + shell: bash + run: | + set -euo pipefail + + COV_FILE="src/unittest/coverage.info" + if [ ! -f "$COV_FILE" ]; then + echo "coverage.info was not generated at $COV_FILE" + exit 1 + fi + + SUMMARY="$(lcov --summary "$COV_FILE" 2>&1)" + echo "$SUMMARY" > /tmp/coverage_summary.txt + + line_pct="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*\([0-9.]*\)%.*/\1/p' | head -n1)" + line_hit="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*[0-9.]*% (\([0-9]*\) of [0-9]*.*/\1/p' | head -n1)" + line_total="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*[0-9.]*% ([0-9]* of \([0-9]*\).*/\1/p' | head -n1)" + + func_pct="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*\([0-9.]*\)%.*/\1/p' | head -n1)" + func_hit="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*[0-9.]*% (\([0-9]*\) of [0-9]*.*/\1/p' | head -n1)" + func_total="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*[0-9.]*% ([0-9]* of \([0-9]*\).*/\1/p' | head -n1)" + + if [ -z "$line_pct" ] || [ -z "$func_pct" ]; then + echo "Failed to parse line/function coverage from lcov summary" + echo "$SUMMARY" + exit 1 + fi + + { + echo "line_pct=$line_pct" + echo "line_hit=$line_hit" + echo "line_total=$line_total" + echo "func_pct=$func_pct" + echo "func_hit=$func_hit" + echo "func_total=$func_total" + } >> "$GITHUB_OUTPUT" + + - name: Save L1 metrics artifact + shell: bash + run: | + { + echo "LINE_PCT=${{ steps.cov.outputs.line_pct }}" + echo "LINE_HIT=${{ steps.cov.outputs.line_hit }}" + echo "LINE_TOTAL=${{ steps.cov.outputs.line_total }}" + echo "FUNC_PCT=${{ steps.cov.outputs.func_pct }}" + echo "FUNC_HIT=${{ steps.cov.outputs.func_hit }}" + echo "FUNC_TOTAL=${{ steps.cov.outputs.func_total }}" + } > /tmp/l1_metrics.env + + { + echo "## L1 Coverage Summary" + echo "" + echo "| Metric | Value |" + echo "|---|---:|" + echo "| Line coverage | ${{ steps.cov.outputs.line_pct }}% (${{ steps.cov.outputs.line_hit }} / ${{ steps.cov.outputs.line_total }}) |" + echo "| Function coverage | ${{ steps.cov.outputs.func_pct }}% (${{ steps.cov.outputs.func_hit }} / ${{ steps.cov.outputs.func_total }}) |" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload L1 artifacts + uses: actions/upload-artifact@v4 + with: + name: l1-coverage-artifacts + path: | + src/unittest/coverage.info + /tmp/coverage_summary.txt + /tmp/l1_metrics.env + + l2-coverage: + name: L2 functional coverage metrics + runs-on: ubuntu-latest + + steps: + - name: Checkout remote_debugger + uses: actions/checkout@v4 + with: + path: remote_debugger + + - name: Checkout dependent repository rfc + uses: actions/checkout@v4 + with: + repository: rdkcentral/rfc + path: rfc + + - name: Checkout dependent repository iarmmgrs + uses: actions/checkout@v4 + with: + repository: rdkcentral/iarmmgrs + path: iarmmgrs + + - name: Checkout dependent repository iarmbus + uses: actions/checkout@v4 + with: + repository: rdkcentral/iarmbus + path: iarmbus + + - name: Checkout dependent repository tr69hostif + uses: actions/checkout@v4 + with: + repository: rdkcentral/tr69hostif + path: tr69hostif + + - name: Pull required docker images + run: | + docker pull ghcr.io/rdkcentral/docker-device-mgt-service-test/mockxconf:latest + docker pull ghcr.io/rdkcentral/docker-device-mgt-service-test/native-platform:latest + + - name: Start mockxconf and native-platform containers + run: | + docker run -d --name mockxconf -p 50050:50050 -p 50051:50051 -p 50052:50052 -p 50054:50054 -v ${{ github.workspace }}:/mnt/L2_CONTAINER_SHARED_VOLUME ghcr.io/rdkcentral/docker-device-mgt-service-test/mockxconf:latest + docker run -d --name native-platform --link mockxconf -v ${{ github.workspace }}:/mnt/L2_CONTAINER_SHARED_VOLUME ghcr.io/rdkcentral/docker-device-mgt-service-test/native-platform:latest + + - name: Move dependencies into native-platform container + run: | + docker exec -i native-platform /bin/bash -c "rm -rf /usr/rfc && mv /mnt/L2_CONTAINER_SHARED_VOLUME/rfc /usr/ && mv /mnt/L2_CONTAINER_SHARED_VOLUME/iarmmgrs /usr/ && rm -rf /usr/iarmbus && mv /mnt/L2_CONTAINER_SHARED_VOLUME/iarmbus /usr/ && mv /mnt/L2_CONTAINER_SHARED_VOLUME/tr69hostif /usr/" + + - name: Build and run L2 tests + run: | + docker exec -i native-platform /bin/bash -c "cd /mnt/L2_CONTAINER_SHARED_VOLUME/remote_debugger && sh cov_build.sh && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/lib/aarch64-linux-gnu:/usr/local/lib: && sh run_l2.sh" + + - name: Copy L2 reports to runner + run: | + docker cp native-platform:/tmp/l2_test_report /tmp/L2_TEST_RESULTS + ls -l /tmp/L2_TEST_RESULTS + + - name: Extract L2 metrics + shell: bash + run: | + set -euo pipefail + + feature_files=$(find remote_debugger/test/functional-tests/features -name '*.feature' | wc -l | tr -d ' ') + feature_scenarios=$(grep -R "^[[:space:]]*Scenario:" remote_debugger/test/functional-tests/features/*.feature | wc -l | tr -d ' ') + feature_scenarios_disabled=$(grep -R "^[[:space:]]*#[[:space:]]*Scenario:" remote_debugger/test/functional-tests/features/*.feature | wc -l | tr -d ' ' || true) + + test_files=$(find remote_debugger/test/functional-tests/tests -maxdepth 1 -name 'test_*.py' | wc -l | tr -d ' ') + test_functions=$(grep -R "^def test_" remote_debugger/test/functional-tests/tests/test_*.py | wc -l | tr -d ' ') + test_functions_disabled=$(grep -R "^[[:space:]]*#[[:space:]]*def test_" remote_debugger/test/functional-tests/tests/test_*.py | wc -l | tr -d ' ' || true) + + src_functions=$(grep -R "^[A-Za-z_][A-Za-z0-9_[:space:]\*]*([^;]*)[[:space:]]*{" remote_debugger/src/*.c | wc -l | tr -d ' ') + + c_api_scenarios=$(grep -n "^[[:space:]]*Scenario:" remote_debugger/test/functional-tests/features/rrd_c_api_upload.feature | wc -l | tr -d ' ') + c_api_tests=$(grep -n "^def test_" remote_debugger/test/functional-tests/tests/test_rrd_c_api_upload.py | wc -l | tr -d ' ') + c_api_gap=$((c_api_scenarios - c_api_tests)) + if [ "$c_api_gap" -lt 0 ]; then + c_api_gap=0 + fi + + functional_coverage_pct=$(awk -v total="$feature_scenarios" -v gap="$c_api_gap" 'BEGIN { if (total > 0) printf "%.2f", ((total-gap)/total)*100; else print "0.00" }') + gap_to_100_pct=$(awk -v cov="$functional_coverage_pct" 'BEGIN { printf "%.2f", 100-cov }') + + python3 - << 'PY' > /tmp/l2_runtime.env +import glob +import json + +totals = {"collected": 0, "passed": 0, "failed": 0, "skipped": 0} +for path in glob.glob('/tmp/L2_TEST_RESULTS/*.json'): + try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + s = data.get('summary', {}) + totals['collected'] += int(s.get('collected', s.get('total', 0) or 0)) + totals['passed'] += int(s.get('passed', 0) or 0) + totals['failed'] += int(s.get('failed', 0) or 0) + totals['skipped'] += int(s.get('skipped', 0) or 0) + except Exception: + continue + +print(f"L2_COLLECTED={totals['collected']}") +print(f"L2_PASSED={totals['passed']}") +print(f"L2_FAILED={totals['failed']}") +print(f"L2_SKIPPED={totals['skipped']}") +PY + + { + echo "FEATURE_FILES=$feature_files" + echo "FEATURE_SCENARIOS=$feature_scenarios" + echo "FEATURE_SCENARIOS_DISABLED=$feature_scenarios_disabled" + echo "TEST_FILES=$test_files" + echo "TEST_FUNCTIONS=$test_functions" + echo "TEST_FUNCTIONS_DISABLED=$test_functions_disabled" + echo "SRC_FUNCTIONS_APPROX=$src_functions" + echo "C_API_SCENARIOS=$c_api_scenarios" + echo "C_API_TESTS=$c_api_tests" + echo "C_API_GAP=$c_api_gap" + echo "FUNCTIONAL_COVERAGE_PCT=$functional_coverage_pct" + echo "GAP_TO_100_PCT=$gap_to_100_pct" + cat /tmp/l2_runtime.env + } > /tmp/l2_metrics.env + + { + echo "## L2 Coverage Summary" + echo "" + echo "| Metric | Value |" + echo "|---|---:|" + echo "| Feature files | $feature_files |" + echo "| Feature scenarios | $feature_scenarios |" + echo "| Test files | $test_files |" + echo "| Test functions | $test_functions |" + echo "| L2 collected | $(grep '^L2_COLLECTED=' /tmp/l2_runtime.env | cut -d= -f2) |" + echo "| L2 passed | $(grep '^L2_PASSED=' /tmp/l2_runtime.env | cut -d= -f2) |" + echo "| L2 failed | $(grep '^L2_FAILED=' /tmp/l2_runtime.env | cut -d= -f2) |" + echo "| L2 skipped | $(grep '^L2_SKIPPED=' /tmp/l2_runtime.env | cut -d= -f2) |" + echo "| Estimated functional coverage | $functional_coverage_pct% |" + echo "| Gap to 100% functional coverage | $gap_to_100_pct% |" + echo "| C API scenario gap | $c_api_gap |" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload L2 artifacts + uses: actions/upload-artifact@v4 + with: + name: l2-coverage-artifacts + path: | + /tmp/L2_TEST_RESULTS + /tmp/l2_metrics.env + + update-docs: + name: Update L1/L2 coverage markdowns + runs-on: ubuntu-latest + needs: [l1-coverage, l2-coverage] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download L1 artifacts + uses: actions/download-artifact@v4 + with: + name: l1-coverage-artifacts + path: /tmp/l1 + + - name: Download L2 artifacts + uses: actions/download-artifact@v4 + with: + name: l2-coverage-artifacts + path: /tmp/l2 + + - name: Regenerate L1 and L2 markdown docs + shell: bash + run: | + set -euo pipefail + source /tmp/l1/l1_metrics.env + source /tmp/l2/l2_metrics.env + GENERATED_DATE="$(date -u +%Y-%m-%d)" + + cat > test/functional-tests/docs/L1_Test_Coverage.md << EOF +# Remote Debugger L1 Coverage Report + +**Generated:** $GENERATED_DATE +**Component:** \`remotedebugger\` +**Source:** PR pipeline coverage run + +--- + +## Executive Summary + +| Metric | Value | +|---|---:| +| Line coverage | $LINE_PCT% ($LINE_HIT / $LINE_TOTAL) | +| Function coverage | $FUNC_PCT% ($FUNC_HIT / $FUNC_TOTAL) | +| Unit test framework | GoogleTest | + +--- + +## Notes + +- Coverage is generated by running \`sh run_ut.sh --enable-cov\`. +- Source artifact: \`src/unittest/coverage.info\`. +- This file is updated automatically on pull requests. +EOF + + cat > test/functional-tests/docs/L2_Test_Coverage.md << EOF +# Remote Debugger L2 Test Coverage Report + +**Generated:** $GENERATED_DATE +**Component:** \`remotedebugger\` +**Test Suite:** \`test/functional-tests\` + +--- + +## Executive Summary + +| Metric | Count | +|---|---:| +| Feature files | $FEATURE_FILES | +| Feature scenarios | $FEATURE_SCENARIOS | +| Disabled feature scenarios | $FEATURE_SCENARIOS_DISABLED | +| Test files (pytest) | $TEST_FILES | +| Test functions (\`def test_*\`) | $TEST_FUNCTIONS | +| Disabled test functions | $TEST_FUNCTIONS_DISABLED | +| Approx source functions | $SRC_FUNCTIONS_APPROX | +| L2 collected tests | $L2_COLLECTED | +| L2 passed tests | $L2_PASSED | +| L2 failed tests | $L2_FAILED | +| L2 skipped tests | $L2_SKIPPED | +| Estimated functional coverage | $FUNCTIONAL_COVERAGE_PCT% | +| Gap to 100% functional coverage | $GAP_TO_100_PCT% | + +--- + +## Gap Highlights + +1. \`rrd_c_api_upload.feature\` scenarios: $C_API_SCENARIOS + \`test_rrd_c_api_upload.py\` test functions: $C_API_TESTS + **Current gap:** $C_API_GAP + +2. Coverage focus areas to improve: + - WebConfig MsgPack decode/error flows + - Upload lock contention and cleanup failure branches + - Startup and RBUS/IARM negative branches + +--- + +## Progress To 100% Functional Coverage + +- Current estimated functional coverage: **$FUNCTIONAL_COVERAGE_PCT%** +- Remaining gap to target (100%): **$GAP_TO_100_PCT%** +- Missing scenario implementations tracked in this run: **$C_API_GAP** + +> Coverage estimate formula used by CI in this report: +> \`((Feature Scenarios - Missing Implemented Scenarios) / Feature Scenarios) * 100\` + +--- + +## Notes + +- L2 results are generated by \`run_l2.sh\` in the PR workflow. +- This file is refreshed automatically on pull requests. +EOF + + - name: Commit docs to PR branch (same-repo PRs only) + id: commit_docs + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + if git diff --quiet -- test/functional-tests/docs/L1_Test_Coverage.md test/functional-tests/docs/L2_Test_Coverage.md; then + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git add test/functional-tests/docs/L1_Test_Coverage.md test/functional-tests/docs/L2_Test_Coverage.md + git commit -m "docs(coverage): refresh L1/L2 coverage for PR #${{ github.event.pull_request.number }}" + git push origin HEAD:${{ github.head_ref }} + echo "changed=true" >> "$GITHUB_OUTPUT" + + - name: Build PR comment body + shell: bash + run: | + set -euo pipefail + source /tmp/l1/l1_metrics.env + source /tmp/l2/l2_metrics.env + DOC_STATUS="updated in PR branch" + if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + DOC_STATUS="not auto-committed (fork PR)" + elif [ "${{ steps.commit_docs.outputs.changed || 'false' }}" = "false" ]; then + DOC_STATUS="no doc changes detected" + fi + + cat > /tmp/pr_coverage_comment.md << EOF + +## PR Coverage Summary + +| Metric | Value | +|---|---:| +| L1 line coverage | $LINE_PCT% ($LINE_HIT / $LINE_TOTAL) | +| L1 function coverage | $FUNC_PCT% ($FUNC_HIT / $FUNC_TOTAL) | +| L2 collected/passed/failed/skipped | $L2_COLLECTED / $L2_PASSED / $L2_FAILED / $L2_SKIPPED | +| L2 feature scenarios | $FEATURE_SCENARIOS | +| L2 test functions | $TEST_FUNCTIONS | +| Estimated functional coverage | $FUNCTIONAL_COVERAGE_PCT% | +| Gap to 100% | $GAP_TO_100_PCT% | +| C API scenario gap | $C_API_GAP | + +Coverage docs status: **$DOC_STATUS** + +Updated files: +- \`test/functional-tests/docs/L1_Test_Coverage.md\` +- \`test/functional-tests/docs/L2_Test_Coverage.md\` +EOF + + cat /tmp/pr_coverage_comment.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upsert PR coverage comment + uses: actions/github-script@v7 + env: + COMMENT_PATH: /tmp/pr_coverage_comment.md + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8'); + const marker = ''; + const issue_number = context.issue.number; + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number + }); + + const existing = comments.find(c => + c.user && c.user.type === 'Bot' && c.body && c.body.includes(marker) + ); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number, + body + }); + } From e9d12a18a823aa2ca340b54c719ff36f85adb838 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Wed, 15 Jul 2026 15:33:02 -0400 Subject: [PATCH 02/25] Rename L1_L2_CoverageReport to L1_L2_CoverageReport.yml --- .../workflows/{L1_L2_CoverageReport => L1_L2_CoverageReport.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{L1_L2_CoverageReport => L1_L2_CoverageReport.yml} (100%) diff --git a/.github/workflows/L1_L2_CoverageReport b/.github/workflows/L1_L2_CoverageReport.yml similarity index 100% rename from .github/workflows/L1_L2_CoverageReport rename to .github/workflows/L1_L2_CoverageReport.yml From f558f4e288b5334d96ead4789cf46df50d766582 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Wed, 15 Jul 2026 15:43:10 -0400 Subject: [PATCH 03/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 44 +++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 6ec60739..4c8f8bdd 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -172,28 +172,28 @@ jobs: functional_coverage_pct=$(awk -v total="$feature_scenarios" -v gap="$c_api_gap" 'BEGIN { if (total > 0) printf "%.2f", ((total-gap)/total)*100; else print "0.00" }') gap_to_100_pct=$(awk -v cov="$functional_coverage_pct" 'BEGIN { printf "%.2f", 100-cov }') - python3 - << 'PY' > /tmp/l2_runtime.env -import glob -import json - -totals = {"collected": 0, "passed": 0, "failed": 0, "skipped": 0} -for path in glob.glob('/tmp/L2_TEST_RESULTS/*.json'): - try: - with open(path, 'r', encoding='utf-8') as f: - data = json.load(f) - s = data.get('summary', {}) - totals['collected'] += int(s.get('collected', s.get('total', 0) or 0)) - totals['passed'] += int(s.get('passed', 0) or 0) - totals['failed'] += int(s.get('failed', 0) or 0) - totals['skipped'] += int(s.get('skipped', 0) or 0) - except Exception: - continue - -print(f"L2_COLLECTED={totals['collected']}") -print(f"L2_PASSED={totals['passed']}") -print(f"L2_FAILED={totals['failed']}") -print(f"L2_SKIPPED={totals['skipped']}") -PY + python3 - <<'PY' > /tmp/l2_runtime.env + import glob + import json + + totals = {"collected": 0, "passed": 0, "failed": 0, "skipped": 0} + for path in glob.glob("/tmp/L2_TEST_RESULTS/*.json"): + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + s = data.get("summary", {}) + totals["collected"] += int(s.get("collected", s.get("total", 0) or 0)) + totals["passed"] += int(s.get("passed", 0) or 0) + totals["failed"] += int(s.get("failed", 0) or 0) + totals["skipped"] += int(s.get("skipped", 0) or 0) + except Exception: + continue + + print(f"L2_COLLECTED={totals['collected']}") + print(f"L2_PASSED={totals['passed']}") + print(f"L2_FAILED={totals['failed']}") + print(f"L2_SKIPPED={totals['skipped']}") + PY { echo "FEATURE_FILES=$feature_files" From cd9e0b6f8ad5fd981041552a7c3366c3c5129de0 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Thu, 16 Jul 2026 15:54:10 -0400 Subject: [PATCH 04/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 168 ++++++++++----------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 4c8f8bdd..c7cc49f4 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -1,4 +1,4 @@ -name: L1 and L2 Coverage Report +name: PR L1 and L2 Coverage Report on: pull_request: @@ -172,28 +172,28 @@ jobs: functional_coverage_pct=$(awk -v total="$feature_scenarios" -v gap="$c_api_gap" 'BEGIN { if (total > 0) printf "%.2f", ((total-gap)/total)*100; else print "0.00" }') gap_to_100_pct=$(awk -v cov="$functional_coverage_pct" 'BEGIN { printf "%.2f", 100-cov }') - python3 - <<'PY' > /tmp/l2_runtime.env - import glob - import json - - totals = {"collected": 0, "passed": 0, "failed": 0, "skipped": 0} - for path in glob.glob("/tmp/L2_TEST_RESULTS/*.json"): - try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - s = data.get("summary", {}) - totals["collected"] += int(s.get("collected", s.get("total", 0) or 0)) - totals["passed"] += int(s.get("passed", 0) or 0) - totals["failed"] += int(s.get("failed", 0) or 0) - totals["skipped"] += int(s.get("skipped", 0) or 0) - except Exception: - continue - - print(f"L2_COLLECTED={totals['collected']}") - print(f"L2_PASSED={totals['passed']}") - print(f"L2_FAILED={totals['failed']}") - print(f"L2_SKIPPED={totals['skipped']}") - PY + python3 -c ' + import glob + import json + + totals = {"collected": 0, "passed": 0, "failed": 0, "skipped": 0} + for path in glob.glob("/tmp/L2_TEST_RESULTS/*.json"): + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + s = data.get("summary", {}) + totals["collected"] += int(s.get("collected", s.get("total", 0) or 0)) + totals["passed"] += int(s.get("passed", 0) or 0) + totals["failed"] += int(s.get("failed", 0) or 0) + totals["skipped"] += int(s.get("skipped", 0) or 0) + except Exception: + continue + + print(f"L2_COLLECTED={totals['"'"'collected'"'"']}") + print(f"L2_PASSED={totals['"'"'passed'"'"']}") + print(f"L2_FAILED={totals['"'"'failed'"'"']}") + print(f"L2_SKIPPED={totals['"'"'skipped'"'"']}") + ' > /tmp/l2_runtime.env { echo "FEATURE_FILES=$feature_files" @@ -268,90 +268,90 @@ jobs: source /tmp/l2/l2_metrics.env GENERATED_DATE="$(date -u +%Y-%m-%d)" - cat > test/functional-tests/docs/L1_Test_Coverage.md << EOF -# Remote Debugger L1 Coverage Report + cat > test/functional-tests/docs/L1_Test_Coverage.md < test/functional-tests/docs/L2_Test_Coverage.md << EOF -# Remote Debugger L2 Test Coverage Report + cat > test/functional-tests/docs/L2_Test_Coverage.md < Coverage estimate formula used by CI in this report: -> \`((Feature Scenarios - Missing Implemented Scenarios) / Feature Scenarios) * 100\` + > Coverage estimate formula used by CI in this report: + > `((Feature Scenarios - Missing Implemented Scenarios) / Feature Scenarios) * 100` ---- + --- -## Notes + ## Notes -- L2 results are generated by \`run_l2.sh\` in the PR workflow. -- This file is refreshed automatically on pull requests. -EOF + - L2 results are generated by `run_l2.sh` in the PR workflow. + - This file is refreshed automatically on pull requests. + EOF - name: Commit docs to PR branch (same-repo PRs only) id: commit_docs From dbfdc301ac7c521974ef288adfc0d4d4bf0d9537 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Thu, 16 Jul 2026 15:56:42 -0400 Subject: [PATCH 05/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 166 ++++++++++----------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index c7cc49f4..4d4df716 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -172,28 +172,28 @@ jobs: functional_coverage_pct=$(awk -v total="$feature_scenarios" -v gap="$c_api_gap" 'BEGIN { if (total > 0) printf "%.2f", ((total-gap)/total)*100; else print "0.00" }') gap_to_100_pct=$(awk -v cov="$functional_coverage_pct" 'BEGIN { printf "%.2f", 100-cov }') - python3 -c ' - import glob - import json - - totals = {"collected": 0, "passed": 0, "failed": 0, "skipped": 0} - for path in glob.glob("/tmp/L2_TEST_RESULTS/*.json"): - try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - s = data.get("summary", {}) - totals["collected"] += int(s.get("collected", s.get("total", 0) or 0)) - totals["passed"] += int(s.get("passed", 0) or 0) - totals["failed"] += int(s.get("failed", 0) or 0) - totals["skipped"] += int(s.get("skipped", 0) or 0) - except Exception: - continue - - print(f"L2_COLLECTED={totals['"'"'collected'"'"']}") - print(f"L2_PASSED={totals['"'"'passed'"'"']}") - print(f"L2_FAILED={totals['"'"'failed'"'"']}") - print(f"L2_SKIPPED={totals['"'"'skipped'"'"']}") - ' > /tmp/l2_runtime.env + python3 - <<'PY' > /tmp/l2_runtime.env + import glob + import json + + totals = {"collected": 0, "passed": 0, "failed": 0, "skipped": 0} + for path in glob.glob("/tmp/L2_TEST_RESULTS/*.json"): + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + s = data.get("summary", {}) + totals["collected"] += int(s.get("collected", s.get("total", 0) or 0)) + totals["passed"] += int(s.get("passed", 0) or 0) + totals["failed"] += int(s.get("failed", 0) or 0) + totals["skipped"] += int(s.get("skipped", 0) or 0) + except Exception: + continue + + print(f"L2_COLLECTED={totals['collected']}") + print(f"L2_PASSED={totals['passed']}") + print(f"L2_FAILED={totals['failed']}") + print(f"L2_SKIPPED={totals['skipped']}") + PY { echo "FEATURE_FILES=$feature_files" @@ -268,90 +268,90 @@ jobs: source /tmp/l2/l2_metrics.env GENERATED_DATE="$(date -u +%Y-%m-%d)" - cat > test/functional-tests/docs/L1_Test_Coverage.md < test/functional-tests/docs/L1_Test_Coverage.md < test/functional-tests/docs/L2_Test_Coverage.md < test/functional-tests/docs/L2_Test_Coverage.md < Coverage estimate formula used by CI in this report: - > `((Feature Scenarios - Missing Implemented Scenarios) / Feature Scenarios) * 100` + > Coverage estimate formula used by CI in this report: + > `((Feature Scenarios - Missing Implemented Scenarios) / Feature Scenarios) * 100` - --- + --- - ## Notes + ## Notes - - L2 results are generated by `run_l2.sh` in the PR workflow. - - This file is refreshed automatically on pull requests. - EOF + - L2 results are generated by `run_l2.sh` in the PR workflow. + - This file is refreshed automatically on pull requests. + EOF - name: Commit docs to PR branch (same-repo PRs only) id: commit_docs From 82ddb48531820aeba62d42ad328c395ce93d41d0 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Thu, 16 Jul 2026 15:58:48 -0400 Subject: [PATCH 06/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 4d4df716..180a6847 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -1,4 +1,4 @@ -name: PR L1 and L2 Coverage Report +name: L1 and L2 Coverage Report on: pull_request: @@ -386,26 +386,26 @@ jobs: fi cat > /tmp/pr_coverage_comment.md << EOF - -## PR Coverage Summary - -| Metric | Value | -|---|---:| -| L1 line coverage | $LINE_PCT% ($LINE_HIT / $LINE_TOTAL) | -| L1 function coverage | $FUNC_PCT% ($FUNC_HIT / $FUNC_TOTAL) | -| L2 collected/passed/failed/skipped | $L2_COLLECTED / $L2_PASSED / $L2_FAILED / $L2_SKIPPED | -| L2 feature scenarios | $FEATURE_SCENARIOS | -| L2 test functions | $TEST_FUNCTIONS | -| Estimated functional coverage | $FUNCTIONAL_COVERAGE_PCT% | -| Gap to 100% | $GAP_TO_100_PCT% | -| C API scenario gap | $C_API_GAP | - -Coverage docs status: **$DOC_STATUS** - -Updated files: -- \`test/functional-tests/docs/L1_Test_Coverage.md\` -- \`test/functional-tests/docs/L2_Test_Coverage.md\` -EOF + + ## PR Coverage Summary + + | Metric | Value | + |---|---:| + | L1 line coverage | $LINE_PCT% ($LINE_HIT / $LINE_TOTAL) | + | L1 function coverage | $FUNC_PCT% ($FUNC_HIT / $FUNC_TOTAL) | + | L2 collected/passed/failed/skipped | $L2_COLLECTED / $L2_PASSED / $L2_FAILED / $L2_SKIPPED | + | L2 feature scenarios | $FEATURE_SCENARIOS | + | L2 test functions | $TEST_FUNCTIONS | + | Estimated functional coverage | $FUNCTIONAL_COVERAGE_PCT% | + | Gap to 100% | $GAP_TO_100_PCT% | + | C API scenario gap | $C_API_GAP | + + Coverage docs status: **$DOC_STATUS** + + Updated files: + - \`test/functional-tests/docs/L1_Test_Coverage.md\` + - \`test/functional-tests/docs/L2_Test_Coverage.md\` + EOF cat /tmp/pr_coverage_comment.md >> "$GITHUB_STEP_SUMMARY" From 36c88120a7177beb6d764317972f38358aba48b6 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Mon, 20 Jul 2026 15:46:33 -0400 Subject: [PATCH 07/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 180a6847..69493038 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -28,7 +28,7 @@ jobs: run: | set -euo pipefail - COV_FILE="src/unittest/coverage.info" + COV_FILE="rfcMgr/gtest/coverage.info" if [ ! -f "$COV_FILE" ]; then echo "coverage.info was not generated at $COV_FILE" exit 1 @@ -86,7 +86,7 @@ jobs: with: name: l1-coverage-artifacts path: | - src/unittest/coverage.info + rfcMgr/gtest/coverage.info /tmp/coverage_summary.txt /tmp/l1_metrics.env @@ -144,7 +144,7 @@ jobs: - name: Copy L2 reports to runner run: | - docker cp native-platform:/tmp/l2_test_report /tmp/L2_TEST_RESULTS + docker cp native-platform:/tmp/rfc_test_report /tmp/L2_TEST_RESULTS ls -l /tmp/L2_TEST_RESULTS - name: Extract L2 metrics @@ -290,7 +290,7 @@ jobs: ## Notes - Coverage is generated by running `sh run_ut.sh --enable-cov`. - - Source artifact: `src/unittest/coverage.info`. + - Source artifact: `rfcMgr/gtest/coverage.info`. - This file is updated automatically on pull requests. EOF From 39d4401975e85bf79f3d361b04be028e64165689 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Tue, 21 Jul 2026 10:46:26 -0400 Subject: [PATCH 08/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 89 +++++++--------------- 1 file changed, 26 insertions(+), 63 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 69493038..69474015 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -95,34 +95,8 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout remote_debugger - uses: actions/checkout@v4 - with: - path: remote_debugger - - - name: Checkout dependent repository rfc - uses: actions/checkout@v4 - with: - repository: rdkcentral/rfc - path: rfc - - - name: Checkout dependent repository iarmmgrs - uses: actions/checkout@v4 - with: - repository: rdkcentral/iarmmgrs - path: iarmmgrs - - - name: Checkout dependent repository iarmbus - uses: actions/checkout@v4 - with: - repository: rdkcentral/iarmbus - path: iarmbus - - - name: Checkout dependent repository tr69hostif + - name: Checkout code uses: actions/checkout@v4 - with: - repository: rdkcentral/tr69hostif - path: tr69hostif - name: Pull required docker images run: | @@ -134,13 +108,9 @@ jobs: docker run -d --name mockxconf -p 50050:50050 -p 50051:50051 -p 50052:50052 -p 50054:50054 -v ${{ github.workspace }}:/mnt/L2_CONTAINER_SHARED_VOLUME ghcr.io/rdkcentral/docker-device-mgt-service-test/mockxconf:latest docker run -d --name native-platform --link mockxconf -v ${{ github.workspace }}:/mnt/L2_CONTAINER_SHARED_VOLUME ghcr.io/rdkcentral/docker-device-mgt-service-test/native-platform:latest - - name: Move dependencies into native-platform container - run: | - docker exec -i native-platform /bin/bash -c "rm -rf /usr/rfc && mv /mnt/L2_CONTAINER_SHARED_VOLUME/rfc /usr/ && mv /mnt/L2_CONTAINER_SHARED_VOLUME/iarmmgrs /usr/ && rm -rf /usr/iarmbus && mv /mnt/L2_CONTAINER_SHARED_VOLUME/iarmbus /usr/ && mv /mnt/L2_CONTAINER_SHARED_VOLUME/tr69hostif /usr/" - - name: Build and run L2 tests run: | - docker exec -i native-platform /bin/bash -c "cd /mnt/L2_CONTAINER_SHARED_VOLUME/remote_debugger && sh cov_build.sh && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/lib/aarch64-linux-gnu:/usr/local/lib: && sh run_l2.sh" + docker exec -i native-platform /bin/bash -c "cd /mnt/L2_CONTAINER_SHARED_VOLUME && sh cov_build.sh && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/lib/aarch64-linux-gnu:/usr/local/lib && sh run_l2.sh" - name: Copy L2 reports to runner run: | @@ -152,22 +122,17 @@ jobs: run: | set -euo pipefail - feature_files=$(find remote_debugger/test/functional-tests/features -name '*.feature' | wc -l | tr -d ' ') - feature_scenarios=$(grep -R "^[[:space:]]*Scenario:" remote_debugger/test/functional-tests/features/*.feature | wc -l | tr -d ' ') - feature_scenarios_disabled=$(grep -R "^[[:space:]]*#[[:space:]]*Scenario:" remote_debugger/test/functional-tests/features/*.feature | wc -l | tr -d ' ' || true) + feature_files=$(find test/functional-tests/features -name '*.feature' | wc -l | tr -d ' ') + feature_scenarios=$(grep -R "^[[:space:]]*Scenario:" test/functional-tests/features/ | wc -l | tr -d ' ') + feature_scenarios_disabled=$(grep -R "^[[:space:]]*#[[:space:]]*Scenario:" test/functional-tests/features/ | wc -l | tr -d ' ' || true) - test_files=$(find remote_debugger/test/functional-tests/tests -maxdepth 1 -name 'test_*.py' | wc -l | tr -d ' ') - test_functions=$(grep -R "^def test_" remote_debugger/test/functional-tests/tests/test_*.py | wc -l | tr -d ' ') - test_functions_disabled=$(grep -R "^[[:space:]]*#[[:space:]]*def test_" remote_debugger/test/functional-tests/tests/test_*.py | wc -l | tr -d ' ' || true) + test_files=$(find test/functional-tests/tests -maxdepth 1 -name 'test_*.py' | wc -l | tr -d ' ') + test_functions=$(grep -R "^def test_" test/functional-tests/tests/ | wc -l | tr -d ' ') + test_functions_disabled=$(grep -R "^[[:space:]]*#[[:space:]]*def test_" test/functional-tests/tests/ | wc -l | tr -d ' ' || true) - src_functions=$(grep -R "^[A-Za-z_][A-Za-z0-9_[:space:]\*]*([^;]*)[[:space:]]*{" remote_debugger/src/*.c | wc -l | tr -d ' ') + src_functions=$(grep -R "^[A-Za-z_][A-Za-z0-9_[:space:]\*]*([^;]*)[[:space:]]*{" rfcMgr/ rfcapi/ tr181api/ utils/ --include='*.cpp' --include='*.c' | wc -l | tr -d ' ') - c_api_scenarios=$(grep -n "^[[:space:]]*Scenario:" remote_debugger/test/functional-tests/features/rrd_c_api_upload.feature | wc -l | tr -d ' ') - c_api_tests=$(grep -n "^def test_" remote_debugger/test/functional-tests/tests/test_rrd_c_api_upload.py | wc -l | tr -d ' ') - c_api_gap=$((c_api_scenarios - c_api_tests)) - if [ "$c_api_gap" -lt 0 ]; then - c_api_gap=0 - fi + c_api_gap=0 functional_coverage_pct=$(awk -v total="$feature_scenarios" -v gap="$c_api_gap" 'BEGIN { if (total > 0) printf "%.2f", ((total-gap)/total)*100; else print "0.00" }') gap_to_100_pct=$(awk -v cov="$functional_coverage_pct" 'BEGIN { printf "%.2f", 100-cov }') @@ -268,11 +233,12 @@ jobs: source /tmp/l2/l2_metrics.env GENERATED_DATE="$(date -u +%Y-%m-%d)" - cat > test/functional-tests/docs/L1_Test_Coverage.md < test/docs/L1_Test_Coverage.md < test/functional-tests/docs/L2_Test_Coverage.md < test/docs/L2_Test_Coverage.md <> "$GITHUB_OUTPUT" exit 0 fi - git add test/functional-tests/docs/L1_Test_Coverage.md test/functional-tests/docs/L2_Test_Coverage.md + git add test/docs/L1_Test_Coverage.md test/docs/L2_Test_Coverage.md git commit -m "docs(coverage): refresh L1/L2 coverage for PR #${{ github.event.pull_request.number }}" git push origin HEAD:${{ github.head_ref }} echo "changed=true" >> "$GITHUB_OUTPUT" @@ -403,8 +366,8 @@ jobs: Coverage docs status: **$DOC_STATUS** Updated files: - - \`test/functional-tests/docs/L1_Test_Coverage.md\` - - \`test/functional-tests/docs/L2_Test_Coverage.md\` + - \`test/docs/L1_Test_Coverage.md\` + - \`test/docs/L2_Test_Coverage.md\` EOF cat /tmp/pr_coverage_comment.md >> "$GITHUB_STEP_SUMMARY" From 8af2becb75023149cf60a7c0df40429824dc648f Mon Sep 17 00:00:00 2001 From: nhanasi Date: Tue, 21 Jul 2026 14:25:07 -0400 Subject: [PATCH 09/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 69474015..ceddf049 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -168,8 +168,6 @@ jobs: echo "TEST_FUNCTIONS=$test_functions" echo "TEST_FUNCTIONS_DISABLED=$test_functions_disabled" echo "SRC_FUNCTIONS_APPROX=$src_functions" - echo "C_API_SCENARIOS=$c_api_scenarios" - echo "C_API_TESTS=$c_api_tests" echo "C_API_GAP=$c_api_gap" echo "FUNCTIONAL_COVERAGE_PCT=$functional_coverage_pct" echo "GAP_TO_100_PCT=$gap_to_100_pct" From 701dbcc448ba496c7e0ee88fd9aef86ea0f63d26 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Wed, 22 Jul 2026 11:34:31 -0400 Subject: [PATCH 10/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index ceddf049..d3cee5fc 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -132,11 +132,7 @@ jobs: src_functions=$(grep -R "^[A-Za-z_][A-Za-z0-9_[:space:]\*]*([^;]*)[[:space:]]*{" rfcMgr/ rfcapi/ tr181api/ utils/ --include='*.cpp' --include='*.c' | wc -l | tr -d ' ') - c_api_gap=0 - - functional_coverage_pct=$(awk -v total="$feature_scenarios" -v gap="$c_api_gap" 'BEGIN { if (total > 0) printf "%.2f", ((total-gap)/total)*100; else print "0.00" }') - gap_to_100_pct=$(awk -v cov="$functional_coverage_pct" 'BEGIN { printf "%.2f", 100-cov }') - + # Parse L2 JSON results first so pass count can drive coverage formula python3 - <<'PY' > /tmp/l2_runtime.env import glob import json @@ -160,6 +156,15 @@ jobs: print(f"L2_SKIPPED={totals['skipped']}") PY + # Source the results so L2_PASSED is available for the coverage formula + # shellcheck source=/dev/null + source /tmp/l2_runtime.env + + # Coverage = tests that actually passed / total feature scenarios + # (not 100% just because every scenario has a test function) + functional_coverage_pct=$(awk -v passed="$L2_PASSED" -v total="$feature_scenarios" 'BEGIN { if (total > 0) printf "%.2f", (passed/total)*100; else print "0.00" }') + gap_to_100_pct=$(awk -v cov="$functional_coverage_pct" 'BEGIN { printf "%.2f", 100-cov }') + { echo "FEATURE_FILES=$feature_files" echo "FEATURE_SCENARIOS=$feature_scenarios" @@ -168,7 +173,6 @@ jobs: echo "TEST_FUNCTIONS=$test_functions" echo "TEST_FUNCTIONS_DISABLED=$test_functions_disabled" echo "SRC_FUNCTIONS_APPROX=$src_functions" - echo "C_API_GAP=$c_api_gap" echo "FUNCTIONAL_COVERAGE_PCT=$functional_coverage_pct" echo "GAP_TO_100_PCT=$gap_to_100_pct" cat /tmp/l2_runtime.env @@ -183,13 +187,12 @@ jobs: echo "| Feature scenarios | $feature_scenarios |" echo "| Test files | $test_files |" echo "| Test functions | $test_functions |" - echo "| L2 collected | $(grep '^L2_COLLECTED=' /tmp/l2_runtime.env | cut -d= -f2) |" - echo "| L2 passed | $(grep '^L2_PASSED=' /tmp/l2_runtime.env | cut -d= -f2) |" - echo "| L2 failed | $(grep '^L2_FAILED=' /tmp/l2_runtime.env | cut -d= -f2) |" - echo "| L2 skipped | $(grep '^L2_SKIPPED=' /tmp/l2_runtime.env | cut -d= -f2) |" + echo "| L2 collected | $L2_COLLECTED |" + echo "| L2 passed | $L2_PASSED |" + echo "| L2 failed | $L2_FAILED |" + echo "| L2 skipped | $L2_SKIPPED |" echo "| Estimated functional coverage | $functional_coverage_pct% |" echo "| Gap to 100% functional coverage | $gap_to_100_pct% |" - echo "| C API scenario gap | $c_api_gap |" } >> "$GITHUB_STEP_SUMMARY" - name: Upload L2 artifacts @@ -301,10 +304,10 @@ jobs: - Current estimated functional coverage: **$FUNCTIONAL_COVERAGE_PCT%** - Remaining gap to target (100%): **$GAP_TO_100_PCT%** - - Missing scenario implementations tracked in this run: **$C_API_GAP** + - L2 failed tests: **$L2_FAILED** > Coverage estimate formula used by CI in this report: - > `((Feature Scenarios - Missing Implemented Scenarios) / Feature Scenarios) * 100` + > `(L2 Passed Tests / Feature Scenarios) * 100` --- @@ -359,7 +362,6 @@ jobs: | L2 test functions | $TEST_FUNCTIONS | | Estimated functional coverage | $FUNCTIONAL_COVERAGE_PCT% | | Gap to 100% | $GAP_TO_100_PCT% | - | C API scenario gap | $C_API_GAP | Coverage docs status: **$DOC_STATUS** From 46c58f466a3b410b8a8eb5bdfbf7acba1fa8fa36 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Wed, 22 Jul 2026 12:05:38 -0400 Subject: [PATCH 11/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index d3cee5fc..e987ac46 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -70,7 +70,7 @@ jobs: echo "FUNC_PCT=${{ steps.cov.outputs.func_pct }}" echo "FUNC_HIT=${{ steps.cov.outputs.func_hit }}" echo "FUNC_TOTAL=${{ steps.cov.outputs.func_total }}" - } > /tmp/l1_metrics.env + } > l1_metrics.env { echo "## L1 Coverage Summary" @@ -88,7 +88,7 @@ jobs: path: | rfcMgr/gtest/coverage.info /tmp/coverage_summary.txt - /tmp/l1_metrics.env + l1_metrics.env l2-coverage: name: L2 functional coverage metrics @@ -176,7 +176,7 @@ jobs: echo "FUNCTIONAL_COVERAGE_PCT=$functional_coverage_pct" echo "GAP_TO_100_PCT=$gap_to_100_pct" cat /tmp/l2_runtime.env - } > /tmp/l2_metrics.env + } > l2_metrics.env { echo "## L2 Coverage Summary" @@ -201,7 +201,7 @@ jobs: name: l2-coverage-artifacts path: | /tmp/L2_TEST_RESULTS - /tmp/l2_metrics.env + l2_metrics.env update-docs: name: Update L1/L2 coverage markdowns From 56c3a98bd77095b8d6f3bf13589ac894ffa5d5b1 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Wed, 22 Jul 2026 14:25:41 -0400 Subject: [PATCH 12/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index e987ac46..15688452 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -218,20 +218,20 @@ jobs: uses: actions/download-artifact@v4 with: name: l1-coverage-artifacts - path: /tmp/l1 + path: ci-artifacts/l1 - name: Download L2 artifacts uses: actions/download-artifact@v4 with: name: l2-coverage-artifacts - path: /tmp/l2 + path: ci-artifacts/l2 - name: Regenerate L1 and L2 markdown docs shell: bash run: | set -euo pipefail - source /tmp/l1/l1_metrics.env - source /tmp/l2/l2_metrics.env + source ci-artifacts/l1/l1_metrics.env + source ci-artifacts/l2/l2_metrics.env GENERATED_DATE="$(date -u +%Y-%m-%d)" mkdir -p test/docs @@ -340,8 +340,8 @@ jobs: shell: bash run: | set -euo pipefail - source /tmp/l1/l1_metrics.env - source /tmp/l2/l2_metrics.env + source ci-artifacts/l1/l1_metrics.env + source ci-artifacts/l2/l2_metrics.env DOC_STATUS="updated in PR branch" if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then DOC_STATUS="not auto-committed (fork PR)" From 39479e3d781671081f99a51bd86976fc6db2bf0c Mon Sep 17 00:00:00 2001 From: nhanasi Date: Thu, 23 Jul 2026 11:48:44 -0400 Subject: [PATCH 13/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 15688452..23fd11e2 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -214,16 +214,16 @@ jobs: with: fetch-depth: 0 - - name: Download L1 artifacts + - name: Download L1 metrics uses: actions/download-artifact@v4 with: - name: l1-coverage-artifacts + name: l1-metrics path: ci-artifacts/l1 - - name: Download L2 artifacts + - name: Download L2 metrics uses: actions/download-artifact@v4 with: - name: l2-coverage-artifacts + name: l2-metrics path: ci-artifacts/l2 - name: Regenerate L1 and L2 markdown docs From 40ea4f412d4b16f4ddaa605e75567bcc683a1a9c Mon Sep 17 00:00:00 2001 From: nhanasi Date: Thu, 23 Jul 2026 15:52:01 -0400 Subject: [PATCH 14/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 23fd11e2..b42b8282 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -70,7 +70,7 @@ jobs: echo "FUNC_PCT=${{ steps.cov.outputs.func_pct }}" echo "FUNC_HIT=${{ steps.cov.outputs.func_hit }}" echo "FUNC_TOTAL=${{ steps.cov.outputs.func_total }}" - } > l1_metrics.env + } > "$GITHUB_WORKSPACE/l1_metrics.env" { echo "## L1 Coverage Summary" @@ -88,7 +88,14 @@ jobs: path: | rfcMgr/gtest/coverage.info /tmp/coverage_summary.txt - l1_metrics.env + if-no-files-found: warn + + - name: Upload L1 metrics + uses: actions/upload-artifact@v4 + with: + name: l1-metrics + path: ${{ github.workspace }}/l1_metrics.env + if-no-files-found: error l2-coverage: name: L2 functional coverage metrics @@ -199,9 +206,14 @@ jobs: uses: actions/upload-artifact@v4 with: name: l2-coverage-artifacts - path: | - /tmp/L2_TEST_RESULTS - l2_metrics.env + path: /tmp/L2_TEST_RESULTS + if-no-files-found: warn + + - name: Upload L2 metrics + uses: actions/upload-artifact@v4 + with: + name: l2-metrics + path: l2_metrics.env update-docs: name: Update L1/L2 coverage markdowns From a2e91a9aa5f937292f5308bdbaf48a48c40c61cf Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 11:44:36 -0400 Subject: [PATCH 15/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index b42b8282..4cc7b5db 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -251,7 +251,7 @@ jobs: # RFC L1 Coverage Report **Generated:** $GENERATED_DATE - **Component:** `rfc` + **Component:** \`rfc\` **Source:** PR pipeline coverage run --- @@ -268,8 +268,8 @@ jobs: ## Notes - - Coverage is generated by running `sh run_ut.sh --enable-cov`. - - Source artifact: `rfcMgr/gtest/coverage.info`. + - Coverage is generated by running \`sh run_ut.sh --enable-cov\`. + - Source artifact: \`rfcMgr/gtest/coverage.info\`. - This file is updated automatically on pull requests. EOF @@ -277,8 +277,8 @@ jobs: # RFC L2 Test Coverage Report **Generated:** $GENERATED_DATE - **Component:** `rfc` - **Test Suite:** `test/functional-tests` + **Component:** \`rfc\` + **Test Suite:** \`test/functional-tests\` --- @@ -290,7 +290,7 @@ jobs: | Feature scenarios | $FEATURE_SCENARIOS | | Disabled feature scenarios | $FEATURE_SCENARIOS_DISABLED | | Test files (pytest) | $TEST_FILES | - | Test functions (`def test_*`) | $TEST_FUNCTIONS | + | Test functions (\`def test_*\`) | $TEST_FUNCTIONS | | Disabled test functions | $TEST_FUNCTIONS_DISABLED | | Approx source functions | $SRC_FUNCTIONS_APPROX | | L2 collected tests | $L2_COLLECTED | @@ -319,13 +319,13 @@ jobs: - L2 failed tests: **$L2_FAILED** > Coverage estimate formula used by CI in this report: - > `(L2 Passed Tests / Feature Scenarios) * 100` + > \`(L2 Passed Tests / Feature Scenarios) * 100\` --- ## Notes - - L2 results are generated by `run_l2.sh` in the PR workflow. + - L2 results are generated by \`run_l2.sh\` in the PR workflow. - This file is refreshed automatically on pull requests. EOF @@ -338,12 +338,12 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - if git diff --quiet -- test/docs/L1_Test_Coverage.md test/docs/L2_Test_Coverage.md; then + git add test/docs/L1_Test_Coverage.md test/docs/L2_Test_Coverage.md + if git diff --cached --quiet; then echo "changed=false" >> "$GITHUB_OUTPUT" exit 0 fi - git add test/docs/L1_Test_Coverage.md test/docs/L2_Test_Coverage.md git commit -m "docs(coverage): refresh L1/L2 coverage for PR #${{ github.event.pull_request.number }}" git push origin HEAD:${{ github.head_ref }} echo "changed=true" >> "$GITHUB_OUTPUT" From 938bb055fc0cfb5397893e343241a59a1a9028c5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:12:37 +0000 Subject: [PATCH 16/25] docs(coverage): refresh L1/L2 coverage for PR #217 --- test/docs/L1_Test_Coverage.md | 23 +++++++++++++++ test/docs/L2_Test_Coverage.md | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 test/docs/L1_Test_Coverage.md create mode 100644 test/docs/L2_Test_Coverage.md diff --git a/test/docs/L1_Test_Coverage.md b/test/docs/L1_Test_Coverage.md new file mode 100644 index 00000000..ee806ff6 --- /dev/null +++ b/test/docs/L1_Test_Coverage.md @@ -0,0 +1,23 @@ +# RFC L1 Coverage Report + +**Generated:** 2026-07-24 +**Component:** `rfc` +**Source:** PR pipeline coverage run + +--- + +## Executive Summary + +| Metric | Value | +|---|---:| +| Line coverage | 81.6% (1893 / 2320) | +| Function coverage | 93.2% (124 / 133) | +| Unit test framework | GoogleTest | + +--- + +## Notes + +- Coverage is generated by running `sh run_ut.sh --enable-cov`. +- Source artifact: `rfcMgr/gtest/coverage.info`. +- This file is updated automatically on pull requests. diff --git a/test/docs/L2_Test_Coverage.md b/test/docs/L2_Test_Coverage.md new file mode 100644 index 00000000..8ebb7cf7 --- /dev/null +++ b/test/docs/L2_Test_Coverage.md @@ -0,0 +1,53 @@ +# RFC L2 Test Coverage Report + +**Generated:** 2026-07-24 +**Component:** `rfc` +**Test Suite:** `test/functional-tests` + +--- + +## Executive Summary + +| Metric | Count | +|---|---:| +| Feature files | 19 | +| Feature scenarios | 35 | +| Disabled feature scenarios | 0 | +| Test files (pytest) | 19 | +| Test functions (`def test_*`) | 35 | +| Disabled test functions | 0 | +| Approx source functions | 232 | +| L2 collected tests | 30 | +| L2 passed tests | 22 | +| L2 failed tests | 8 | +| L2 skipped tests | 0 | +| Estimated functional coverage | 62.86% | +| Gap to 100% functional coverage | 37.14% | + +--- + +## Gap Highlights + +Coverage focus areas to improve: +- XConf communication error/retry flows +- mTLS certificate selection negative branches +- IARM bus startup and failure branches +- RFC parameter set/get edge cases + +--- + +## Progress To 100% Functional Coverage + +- Current estimated functional coverage: **62.86%** +- Remaining gap to target (100%): **37.14%** +- L2 failed tests: **8** + +> Coverage estimate formula used by CI in this report: +> `(L2 Passed Tests / Feature Scenarios) * 100` + +--- + +## Notes + +- L2 results are generated by `run_l2.sh` in the PR workflow. +- This file is refreshed automatically on pull requests. From 1d87c4cc5ec5a1bdb86951516e6e79a164a8164a Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 13:43:00 -0400 Subject: [PATCH 17/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 171 +++++++++++---------- 1 file changed, 86 insertions(+), 85 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 4cc7b5db..97c9166e 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -246,88 +246,89 @@ jobs: source ci-artifacts/l2/l2_metrics.env GENERATED_DATE="$(date -u +%Y-%m-%d)" - mkdir -p test/docs - cat > test/docs/L1_Test_Coverage.md < test/docs/L2_Test_Coverage.md < Coverage estimate formula used by CI in this report: - > \`(L2 Passed Tests / Feature Scenarios) * 100\` - - --- - - ## Notes - - - L2 results are generated by \`run_l2.sh\` in the PR workflow. - - This file is refreshed automatically on pull requests. - EOF + export GENERATED_DATE LINE_PCT LINE_HIT LINE_TOTAL FUNC_PCT FUNC_HIT FUNC_TOTAL \ + FEATURE_FILES FEATURE_SCENARIOS FEATURE_SCENARIOS_DISABLED TEST_FILES \ + TEST_FUNCTIONS TEST_FUNCTIONS_DISABLED SRC_FUNCTIONS_APPROX \ + L2_COLLECTED L2_PASSED L2_FAILED L2_SKIPPED FUNCTIONAL_COVERAGE_PCT GAP_TO_100_PCT + + python3 - <<'PY' +import os, re + +d = os.environ + +# ---- L1_Analysis_Report.md: replace the top summary section ---- +l1_new_header = ( + "# L1 Code Coverage Report\n\n" + "**Test File:** coverage.info \n" + f"**Date:** {d['GENERATED_DATE']} \n\n" + "## Summary\n" + f"- **Line Coverage:** {d['LINE_HIT']} / {d['LINE_TOTAL']} (**{d['LINE_PCT']}%**)\n" + f"- **Function Coverage:** {d['FUNC_HIT']} / {d['FUNC_TOTAL']} (**{d['FUNC_PCT']}%**)\n\n" +) + +with open('test/docs/L1_Analysis_Report.md', 'r') as f: + content = f.read() + +new_content = re.sub( + r'\A.*?(?=^## Directory Breakdown)', + l1_new_header, + content, count=1, flags=re.DOTALL | re.MULTILINE +) +if new_content == content: + new_content = re.sub(r'\A.*?(?=^---)', l1_new_header, content, count=1, flags=re.DOTALL | re.MULTILINE) +if new_content == content: + new_content = l1_new_header + content + +with open('test/docs/L1_Analysis_Report.md', 'w') as f: + f.write(new_content) +print("L1_Analysis_Report.md updated") + +# ---- L2_Analysis_Report.md: update individual lines inside the ``` block ---- +# Only lines whose values come from the CI run are replaced. +# Static estimates (source function breakdown, proposed scenarios, etc.) are preserved. +LINE_REPLACEMENTS = [ + (r'(Active L2 test functions:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS']), + (r'(Disabled L2 test functions:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), + (r'(Active feature scenarios:\s+)[\d~]+', r'\g<1>' + d['FEATURE_SCENARIOS']), + (r'(Test files active:\s+)[\d~]+', r'\g<1>' + d['TEST_FILES']), + (r'(Test files disabled[^:]*:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), + (r'(Estimated current L2 functional coverage:\s+)~?[\d.]+%', + r'\g<1>' + d['FUNCTIONAL_COVERAGE_PCT'] + '%'), +] + +with open('test/docs/L2_Analysis_Report.md', 'r') as f: + content = f.read() + +# Locate the fenced block and apply replacements only within it +def update_block(m): + block = m.group(0) + for pattern, repl in LINE_REPLACEMENTS: + block = re.sub(pattern, repl, block) + return block + +new_content = re.sub( + r'(?<=\*\*Test Coverage Summary\*\*\n```\n).*?(?=```)', + update_block, + content, count=1, flags=re.DOTALL +) + +# Append/update the generated date line just before the closing ``` +new_content = re.sub( + r'(\n\*Generated:.*?\*)(\n```)', + f'\n*Generated: {d["GENERATED_DATE"]} — L2 passed: {d["L2_PASSED"]}/{d["L2_COLLECTED"]}, failed: {d["L2_FAILED"]}*\\2', + new_content, count=1 +) +if '*Generated:' not in new_content: + new_content = re.sub( + r'((?<=\*\*Test Coverage Summary\*\*\n```\n).*?)(```)', + r'\1' + f'*Generated: {d["GENERATED_DATE"]} — L2 passed: {d["L2_PASSED"]}/{d["L2_COLLECTED"]}, failed: {d["L2_FAILED"]}*\n' + r'```', + new_content, count=1, flags=re.DOTALL + ) + +with open('test/docs/L2_Analysis_Report.md', 'w') as f: + f.write(new_content) +print("L2_Analysis_Report.md updated") +PY - name: Commit docs to PR branch (same-repo PRs only) id: commit_docs @@ -338,7 +339,7 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add test/docs/L1_Test_Coverage.md test/docs/L2_Test_Coverage.md + git add test/docs/L1_Analysis_Report.md test/docs/L2_Analysis_Report.md if git diff --cached --quiet; then echo "changed=false" >> "$GITHUB_OUTPUT" exit 0 @@ -378,8 +379,8 @@ jobs: Coverage docs status: **$DOC_STATUS** Updated files: - - \`test/docs/L1_Test_Coverage.md\` - - \`test/docs/L2_Test_Coverage.md\` + - \`test/docs/L1_Analysis_Report.md\` + - \`test/docs/L2_Analysis_Report.md\` EOF cat /tmp/pr_coverage_comment.md >> "$GITHUB_STEP_SUMMARY" From 6bcec066d728769df6cd995efb595ed9af44b8ee Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 13:47:46 -0400 Subject: [PATCH 18/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 97c9166e..aa24dd75 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -251,7 +251,7 @@ jobs: TEST_FUNCTIONS TEST_FUNCTIONS_DISABLED SRC_FUNCTIONS_APPROX \ L2_COLLECTED L2_PASSED L2_FAILED L2_SKIPPED FUNCTIONAL_COVERAGE_PCT GAP_TO_100_PCT - python3 - <<'PY' + python3 - < Date: Fri, 24 Jul 2026 14:04:38 -0400 Subject: [PATCH 19/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 143 ++++++++++----------- 1 file changed, 66 insertions(+), 77 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index aa24dd75..fbab8c05 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -252,83 +252,72 @@ jobs: L2_COLLECTED L2_PASSED L2_FAILED L2_SKIPPED FUNCTIONAL_COVERAGE_PCT GAP_TO_100_PCT python3 - <' + d['TEST_FUNCTIONS']), - (r'(Disabled L2 test functions:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), - (r'(Active feature scenarios:\s+)[\d~]+', r'\g<1>' + d['FEATURE_SCENARIOS']), - (r'(Test files active:\s+)[\d~]+', r'\g<1>' + d['TEST_FILES']), - (r'(Test files disabled[^:]*:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), - (r'(Estimated current L2 functional coverage:\s+)~?[\d.]+%', - r'\g<1>' + d['FUNCTIONAL_COVERAGE_PCT'] + '%'), -] - -with open('test/docs/L2_Analysis_Report.md', 'r') as f: - content = f.read() - -# Locate the fenced block and apply replacements only within it -def update_block(m): - block = m.group(0) - for pattern, repl in LINE_REPLACEMENTS: - block = re.sub(pattern, repl, block) - return block - -new_content = re.sub( - r'(?<=\*\*Test Coverage Summary\*\*\n```\n).*?(?=```)', - update_block, - content, count=1, flags=re.DOTALL -) - -# Append/update the generated date line just before the closing ``` -new_content = re.sub( - r'(\n\*Generated:.*?\*)(\n```)', - f'\n*Generated: {d["GENERATED_DATE"]} — L2 passed: {d["L2_PASSED"]}/{d["L2_COLLECTED"]}, failed: {d["L2_FAILED"]}*\\2', - new_content, count=1 -) -if '*Generated:' not in new_content: - new_content = re.sub( - r'((?<=\*\*Test Coverage Summary\*\*\n```\n).*?)(```)', - r'\1' + f'*Generated: {d["GENERATED_DATE"]} — L2 passed: {d["L2_PASSED"]}/{d["L2_COLLECTED"]}, failed: {d["L2_FAILED"]}*\n' + r'```', - new_content, count=1, flags=re.DOTALL - ) - -with open('test/docs/L2_Analysis_Report.md', 'w') as f: - f.write(new_content) -print("L2_Analysis_Report.md updated") -PY + import os, re + + d = os.environ + + # ---- L1_Analysis_Report.md: replace the top summary section ---- + l1_new_header = ( + "# L1 Code Coverage Report\n\n" + "**Test File:** coverage.info \n" + f"**Date:** {d['GENERATED_DATE']} \n\n" + "## Summary\n" + f"- **Line Coverage:** {d['LINE_HIT']} / {d['LINE_TOTAL']} (**{d['LINE_PCT']}%**)\n" + f"- **Function Coverage:** {d['FUNC_HIT']} / {d['FUNC_TOTAL']} (**{d['FUNC_PCT']}%**)\n\n" + ) + + with open('test/docs/L1_Analysis_Report.md', 'r') as f: + content = f.read() + + new_content = re.sub( + r'\A.*?(?=^## Directory Breakdown)', + l1_new_header, + content, count=1, flags=re.DOTALL | re.MULTILINE + ) + if new_content == content: + new_content = re.sub(r'\A.*?(?=^---)', l1_new_header, content, count=1, flags=re.DOTALL | re.MULTILINE) + if new_content == content: + new_content = l1_new_header + content + + with open('test/docs/L1_Analysis_Report.md', 'w') as f: + f.write(new_content) + print("L1_Analysis_Report.md updated") + + # ---- L2_Analysis_Report.md: update individual lines inside the ``` block ---- + LINE_REPLACEMENTS = [ + (r'(Active L2 test functions:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS']), + (r'(Disabled L2 test functions:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), + (r'(Active feature scenarios:\s+)[\d~]+', r'\g<1>' + d['FEATURE_SCENARIOS']), + (r'(Test files active:\s+)[\d~]+', r'\g<1>' + d['TEST_FILES']), + (r'(Test files disabled[^:]*:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), + (r'(Estimated current L2 functional coverage:\s+)~?[\d.]+%', + r'\g<1>' + d['FUNCTIONAL_COVERAGE_PCT'] + '%'), + ] + + with open('test/docs/L2_Analysis_Report.md', 'r') as f: + content = f.read() + + def update_block(m): + block = m.group(0) + for pattern, repl in LINE_REPLACEMENTS: + block = re.sub(pattern, repl, block) + return block + + new_content = re.sub( + r'(?<=\*\*Test Coverage Summary\*\*\n```\n).*?(?=```)', + update_block, + content, count=1, flags=re.DOTALL + ) + + gen_line = f'\n*Generated: {d["GENERATED_DATE"]} - L2 passed: {d["L2_PASSED"]}/{d["L2_COLLECTED"]}, failed: {d["L2_FAILED"]}*' + new_content = re.sub(r'\n\*Generated:.*?\*(?=\n```)', gen_line, new_content, count=1) + if '*Generated:' not in new_content: + new_content = re.sub(r'(\n```)', gen_line + r'\1', new_content, count=1) + + with open('test/docs/L2_Analysis_Report.md', 'w') as f: + f.write(new_content) + print("L2_Analysis_Report.md updated") + PY - name: Commit docs to PR branch (same-repo PRs only) id: commit_docs From 4677e85a3021d28ddcd9ea5c0731bcfe915a9dc7 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 15:01:02 -0400 Subject: [PATCH 20/25] Update code-coverage.yml --- .github/workflows/code-coverage.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 141eea71..a12e4523 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -19,6 +19,36 @@ jobs: run: | sh run_ut.sh --enable-cov + - name: Parse coverage and upload l1-metrics + shell: bash + run: | + set -euo pipefail + COV_FILE="rfcMgr/gtest/coverage.info" + SUMMARY="$(lcov --summary "$COV_FILE" 2>&1)" + + line_pct="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*\([0-9.]*\)%.*/\1/p' | head -n1)" + line_hit="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*[0-9.]*% (\([0-9]*\) of [0-9]*.*/\1/p' | head -n1)" + line_total="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*[0-9.]*% ([0-9]* of \([0-9]*\).*/\1/p' | head -n1)" + func_pct="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*\([0-9.]*\)%.*/\1/p' | head -n1)" + func_hit="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*[0-9.]*% (\([0-9]*\) of [0-9]*.*/\1/p' | head -n1)" + func_total="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*[0-9.]*% ([0-9]* of \([0-9]*\).*/\1/p' | head -n1)" + + { + echo "LINE_PCT=$line_pct" + echo "LINE_HIT=$line_hit" + echo "LINE_TOTAL=$line_total" + echo "FUNC_PCT=$func_pct" + echo "FUNC_HIT=$func_hit" + echo "FUNC_TOTAL=$func_total" + } > l1_metrics.env + + - name: Upload l1-metrics + uses: actions/upload-artifact@v4 + with: + name: l1-metrics + path: l1_metrics.env + if-no-files-found: error + - name: Caculate the code coverage summary run: | cd ./rfcMgr/gtest From 39e8b016e5bdadde6d1ee3b8aaa2b7046b985e68 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 15:01:17 -0400 Subject: [PATCH 21/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 248 +++++---------------- 1 file changed, 55 insertions(+), 193 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index fbab8c05..1f3eff43 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -1,102 +1,15 @@ name: L1 and L2 Coverage Report on: - pull_request: - branches: [ develop, main ] + workflow_run: + workflows: ["Code Coverage"] + types: [completed] permissions: contents: write pull-requests: write jobs: - l1-coverage: - name: L1 unit coverage metrics - runs-on: ubuntu-latest - container: - image: ghcr.io/rdkcentral/docker-rdk-ci:latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run unit tests with coverage enabled - run: sh run_ut.sh --enable-cov - - - name: Extract line and function coverage - id: cov - shell: bash - run: | - set -euo pipefail - - COV_FILE="rfcMgr/gtest/coverage.info" - if [ ! -f "$COV_FILE" ]; then - echo "coverage.info was not generated at $COV_FILE" - exit 1 - fi - - SUMMARY="$(lcov --summary "$COV_FILE" 2>&1)" - echo "$SUMMARY" > /tmp/coverage_summary.txt - - line_pct="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*\([0-9.]*\)%.*/\1/p' | head -n1)" - line_hit="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*[0-9.]*% (\([0-9]*\) of [0-9]*.*/\1/p' | head -n1)" - line_total="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*lines\.*:[[:space:]]*[0-9.]*% ([0-9]* of \([0-9]*\).*/\1/p' | head -n1)" - - func_pct="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*\([0-9.]*\)%.*/\1/p' | head -n1)" - func_hit="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*[0-9.]*% (\([0-9]*\) of [0-9]*.*/\1/p' | head -n1)" - func_total="$(echo "$SUMMARY" | sed -n 's/^[[:space:]]*functions\.*:[[:space:]]*[0-9.]*% ([0-9]* of \([0-9]*\).*/\1/p' | head -n1)" - - if [ -z "$line_pct" ] || [ -z "$func_pct" ]; then - echo "Failed to parse line/function coverage from lcov summary" - echo "$SUMMARY" - exit 1 - fi - - { - echo "line_pct=$line_pct" - echo "line_hit=$line_hit" - echo "line_total=$line_total" - echo "func_pct=$func_pct" - echo "func_hit=$func_hit" - echo "func_total=$func_total" - } >> "$GITHUB_OUTPUT" - - - name: Save L1 metrics artifact - shell: bash - run: | - { - echo "LINE_PCT=${{ steps.cov.outputs.line_pct }}" - echo "LINE_HIT=${{ steps.cov.outputs.line_hit }}" - echo "LINE_TOTAL=${{ steps.cov.outputs.line_total }}" - echo "FUNC_PCT=${{ steps.cov.outputs.func_pct }}" - echo "FUNC_HIT=${{ steps.cov.outputs.func_hit }}" - echo "FUNC_TOTAL=${{ steps.cov.outputs.func_total }}" - } > "$GITHUB_WORKSPACE/l1_metrics.env" - - { - echo "## L1 Coverage Summary" - echo "" - echo "| Metric | Value |" - echo "|---|---:|" - echo "| Line coverage | ${{ steps.cov.outputs.line_pct }}% (${{ steps.cov.outputs.line_hit }} / ${{ steps.cov.outputs.line_total }}) |" - echo "| Function coverage | ${{ steps.cov.outputs.func_pct }}% (${{ steps.cov.outputs.func_hit }} / ${{ steps.cov.outputs.func_total }}) |" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Upload L1 artifacts - uses: actions/upload-artifact@v4 - with: - name: l1-coverage-artifacts - path: | - rfcMgr/gtest/coverage.info - /tmp/coverage_summary.txt - if-no-files-found: warn - - - name: Upload L1 metrics - uses: actions/upload-artifact@v4 - with: - name: l1-metrics - path: ${{ github.workspace }}/l1_metrics.env - if-no-files-found: error - l2-coverage: name: L2 functional coverage metrics runs-on: ubuntu-latest @@ -104,6 +17,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha }} - name: Pull required docker images run: | @@ -202,10 +117,10 @@ jobs: echo "| Gap to 100% functional coverage | $gap_to_100_pct% |" } >> "$GITHUB_STEP_SUMMARY" - - name: Upload L2 artifacts + - name: Upload L2 per-test JSON reports uses: actions/upload-artifact@v4 with: - name: l2-coverage-artifacts + name: l2-test-reports path: /tmp/L2_TEST_RESULTS if-no-files-found: warn @@ -218,18 +133,23 @@ jobs: update-docs: name: Update L1/L2 coverage markdowns runs-on: ubuntu-latest - needs: [l1-coverage, l2-coverage] + needs: [l2-coverage] + # Only update docs when code-coverage.yml succeeded (l1-metrics exists) + if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout code uses: actions/checkout@v4 with: + ref: ${{ github.event.workflow_run.head_sha }} fetch-depth: 0 - name: Download L1 metrics uses: actions/download-artifact@v4 with: name: l1-metrics + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} path: ci-artifacts/l1 - name: Download L2 metrics @@ -238,90 +158,24 @@ jobs: name: l2-metrics path: ci-artifacts/l2 + - name: Download L2 per-test JSON reports + uses: actions/download-artifact@v4 + with: + name: l2-test-reports + path: ci-artifacts/l2-reports + - name: Regenerate L1 and L2 markdown docs shell: bash run: | set -euo pipefail - source ci-artifacts/l1/l1_metrics.env - source ci-artifacts/l2/l2_metrics.env - GENERATED_DATE="$(date -u +%Y-%m-%d)" - - export GENERATED_DATE LINE_PCT LINE_HIT LINE_TOTAL FUNC_PCT FUNC_HIT FUNC_TOTAL \ - FEATURE_FILES FEATURE_SCENARIOS FEATURE_SCENARIOS_DISABLED TEST_FILES \ - TEST_FUNCTIONS TEST_FUNCTIONS_DISABLED SRC_FUNCTIONS_APPROX \ - L2_COLLECTED L2_PASSED L2_FAILED L2_SKIPPED FUNCTIONAL_COVERAGE_PCT GAP_TO_100_PCT - - python3 - <' + d['TEST_FUNCTIONS']), - (r'(Disabled L2 test functions:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), - (r'(Active feature scenarios:\s+)[\d~]+', r'\g<1>' + d['FEATURE_SCENARIOS']), - (r'(Test files active:\s+)[\d~]+', r'\g<1>' + d['TEST_FILES']), - (r'(Test files disabled[^:]*:\s+)[\d~]+', r'\g<1>' + d['TEST_FUNCTIONS_DISABLED']), - (r'(Estimated current L2 functional coverage:\s+)~?[\d.]+%', - r'\g<1>' + d['FUNCTIONAL_COVERAGE_PCT'] + '%'), - ] - - with open('test/docs/L2_Analysis_Report.md', 'r') as f: - content = f.read() - - def update_block(m): - block = m.group(0) - for pattern, repl in LINE_REPLACEMENTS: - block = re.sub(pattern, repl, block) - return block - - new_content = re.sub( - r'(?<=\*\*Test Coverage Summary\*\*\n```\n).*?(?=```)', - update_block, - content, count=1, flags=re.DOTALL - ) - - gen_line = f'\n*Generated: {d["GENERATED_DATE"]} - L2 passed: {d["L2_PASSED"]}/{d["L2_COLLECTED"]}, failed: {d["L2_FAILED"]}*' - new_content = re.sub(r'\n\*Generated:.*?\*(?=\n```)', gen_line, new_content, count=1) - if '*Generated:' not in new_content: - new_content = re.sub(r'(\n```)', gen_line + r'\1', new_content, count=1) - - with open('test/docs/L2_Analysis_Report.md', 'w') as f: - f.write(new_content) - print("L2_Analysis_Report.md updated") - PY + python3 test/scripts/update_coverage_docs.py \ + --reports-dir ci-artifacts/l2-reports \ + --l1-env ci-artifacts/l1/l1_metrics.env \ + --l2-env ci-artifacts/l2/l2_metrics.env - name: Commit docs to PR branch (same-repo PRs only) id: commit_docs - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + if: ${{ github.event.workflow_run.head_repository.full_name == github.repository }} shell: bash run: | set -euo pipefail @@ -334,8 +188,8 @@ jobs: exit 0 fi - git commit -m "docs(coverage): refresh L1/L2 coverage for PR #${{ github.event.pull_request.number }}" - git push origin HEAD:${{ github.head_ref }} + git commit -m "docs(coverage): refresh L1/L2 coverage for PR #${{ github.event.workflow_run.pull_requests[0].number }}" + git push origin HEAD:${{ github.event.workflow_run.head_branch }} echo "changed=true" >> "$GITHUB_OUTPUT" - name: Build PR comment body @@ -345,32 +199,38 @@ jobs: source ci-artifacts/l1/l1_metrics.env source ci-artifacts/l2/l2_metrics.env DOC_STATUS="updated in PR branch" - if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + if [ "${{ github.event.workflow_run.head_repository.full_name }}" != "${{ github.repository }}" ]; then DOC_STATUS="not auto-committed (fork PR)" elif [ "${{ steps.commit_docs.outputs.changed || 'false' }}" = "false" ]; then DOC_STATUS="no doc changes detected" fi + # Extract the CI-generated component table from the updated doc + COMPONENT_TABLE=$(python3 - <<'PY' +import re, sys +with open("test/docs/L2_Analysis_Report.md", "r") as f: + content = f.read() +m = re.search(r"\n(.*?)\n", content, re.DOTALL) +print(m.group(1) if m else "_Component table not available_") +PY +) + cat > /tmp/pr_coverage_comment.md << EOF - - ## PR Coverage Summary - - | Metric | Value | - |---|---:| - | L1 line coverage | $LINE_PCT% ($LINE_HIT / $LINE_TOTAL) | - | L1 function coverage | $FUNC_PCT% ($FUNC_HIT / $FUNC_TOTAL) | - | L2 collected/passed/failed/skipped | $L2_COLLECTED / $L2_PASSED / $L2_FAILED / $L2_SKIPPED | - | L2 feature scenarios | $FEATURE_SCENARIOS | - | L2 test functions | $TEST_FUNCTIONS | - | Estimated functional coverage | $FUNCTIONAL_COVERAGE_PCT% | - | Gap to 100% | $GAP_TO_100_PCT% | - - Coverage docs status: **$DOC_STATUS** - - Updated files: - - \`test/docs/L1_Analysis_Report.md\` - - \`test/docs/L2_Analysis_Report.md\` - EOF + +## PR Coverage Summary + +### L1 Code Coverage +| Metric | Value | +|---|---:| +| Line coverage | ${LINE_PCT}% (${LINE_HIT} / ${LINE_TOTAL}) | +| Function coverage | ${FUNC_PCT}% (${FUNC_HIT} / ${FUNC_TOTAL}) | + +### L2 Functional Coverage by Component +${COMPONENT_TABLE} + +Coverage docs status: **${DOC_STATUS}** +Updated: \`test/docs/L1_Analysis_Report.md\` · \`test/docs/L2_Analysis_Report.md\` +EOF cat /tmp/pr_coverage_comment.md >> "$GITHUB_STEP_SUMMARY" @@ -378,12 +238,14 @@ jobs: uses: actions/github-script@v7 env: COMMENT_PATH: /tmp/pr_coverage_comment.md + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} with: script: | const fs = require('fs'); const body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8'); const marker = ''; - const issue_number = context.issue.number; + const issue_number = parseInt(process.env.PR_NUMBER); + if (!issue_number) { console.log('No PR number — skipping comment'); return; } const comments = await github.paginate(github.rest.issues.listComments, { owner: context.repo.owner, From 795d149da263ac53c623a7de67cf00b68dc76cdd Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 15:02:04 -0400 Subject: [PATCH 22/25] Create update_coverage_docs.py --- test/scripts/update_coverage_docs.py | 360 +++++++++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 test/scripts/update_coverage_docs.py diff --git a/test/scripts/update_coverage_docs.py b/test/scripts/update_coverage_docs.py new file mode 100644 index 00000000..c6cda07b --- /dev/null +++ b/test/scripts/update_coverage_docs.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +""" +update_coverage_docs.py — RFC CI coverage doc updater. + +Reads per-test pytest JSON reports plus L1/L2 aggregate env files, +then rewrites the CI-generated sections in: + test/docs/L1_Analysis_Report.md + test/docs/L2_Analysis_Report.md + +Usage: + python3 test/scripts/update_coverage_docs.py \ + --reports-dir ci-artifacts/l2-reports \ + --l1-env ci-artifacts/l1/l1_metrics.env \ + --l2-env ci-artifacts/l2/l2_metrics.env +""" + +import argparse +import glob +import json +import os +import re +import sys + + +# --------------------------------------------------------------------------- +# Component map: JSON report filename stem -> (component, [source_fns], disabled) +# Stems match the filenames produced by run_l2.sh and run_l2_reboot_trigger.sh. +# --------------------------------------------------------------------------- +COMPONENT_MAP = { + "rfc_single_instance_run": ("A. Startup & Init", ["main()", "CurrentRunningInst()"], False), + "rfc_init_failure": ("A. Startup & Init", ["GetServURL()"], False), + "rfc_override_rfc_prop": ("A. Startup & Init", ["GetServURL()"], False), + "rfc_device_offline": ("B. Device Connectivity", ["isDnsResolve()"], False), + "rfc_xconf_communication_success": ("C. XConf Communication", ["ProcessRuntimeFeatureControlReq()", "CreateXconfHTTPUrl()"], False), + "rfc_xconf_request_params": ("C. XConf Communication", ["CreateXconfHTTPUrl()"], False), + "rfc_feature_enable": ("C. XConf Communication", ["ProcessRuntimeFeatureControlReq()"], False), + "rfc_setget_param": ("D. RFC Param Mgmt", ["setRFCParameter()", "getRFCParameter()"], False), + "rfc_tr181_setget_local_param": ("D. RFC Param Mgmt", ["setLocalParam()", "getLocalParam()"], False), + "rfc_factory_reset": ("D. RFC Param Mgmt", ["processXconfResponseConfigDataPart()"], False), + "rfc_valid_accountid": ("E. AccountID Lifecycle", ["GetValidAccountId()"], False), + "rfc_trigger_reboot_unknown_accountid": ("E. AccountID Lifecycle", ["isConfigValueChange()"], False), + "rfc_unknown_accountid": ("E. AccountID Lifecycle", ["rfcCheckAccountId()"], False), + "rfc_xconf_reboot": ("F. Reboot & Maintenance", ["SendEventToMaintenanceManager()"], False), + "rfc_configsethash_time": ("G. Config Tracking", ["updateHashAndTimeInDB()"], False), + "rfc_xconf_rfc_data": ("H. Data Persistence", ["processXconfResponseConfigDataPart()"], False), + "rfc_dynamic_static_cert_selector": ("I. mTLS / Certificate", ["getMtlscert()"], True), + "rfc_static_cert_selector": ("I. mTLS / Certificate", ["getMtlscert()"], True), + "rfc_rfc_webpa": ("J. WebPA", ["IARM event handler"], False), +} + +COMPONENT_ORDER = [ + "A. Startup & Init", + "B. Device Connectivity", + "C. XConf Communication", + "D. RFC Param Mgmt", + "E. AccountID Lifecycle", + "F. Reboot & Maintenance", + "G. Config Tracking", + "H. Data Persistence", + "I. mTLS / Certificate", + "J. WebPA", +] + +# Outcome icons used in per-test detail rows +OUTCOME_ICON = { + "passed": "PASS", + "failed": "FAIL", + "error": "ERROR", + "skipped": "SKIP", + "not_run": "NOT RUN", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def load_env_file(path): + """Parse a KEY=VALUE env file and return a dict.""" + env = {} + if not path or not os.path.isfile(path): + return env + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, _, val = line.partition("=") + env[key.strip()] = val.strip() + return env + + +def load_reports(reports_dir): + """Parse per-test results from every *.json in reports_dir. + + Returns dict {stem: {"summary": {...}, "tests": [{"name":..., "outcome":...}]}} + """ + results = {} + if not reports_dir or not os.path.isdir(reports_dir): + print(f"WARNING: reports directory not found: {reports_dir}", file=sys.stderr) + return results + + for path in glob.glob(os.path.join(reports_dir, "*.json")): + stem = os.path.splitext(os.path.basename(path))[0] + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + tests = [] + for t in data.get("tests", []): + nodeid = t.get("nodeid", "") + func_name = nodeid.split("::")[-1] if "::" in nodeid else nodeid + tests.append({"name": func_name, "outcome": t.get("outcome", "unknown")}) + results[stem] = {"summary": data.get("summary", {}), "tests": tests} + except Exception as exc: + print(f"WARNING: Could not parse {path}: {exc}", file=sys.stderr) + return results + + +def compute_component_metrics(results): + """Group test results by component. + + Returns dict {component_label: {"source_fns": set, "rows": [...]}} + where each row is {"name": str, "outcome": str, "disabled": bool}. + """ + comp_data = {c: {"source_fns": set(), "rows": []} for c in COMPONENT_ORDER} + + for stem, (comp, source_fns, disabled) in COMPONENT_MAP.items(): + for fn in source_fns: + comp_data[comp]["source_fns"].add(fn) + + if stem in results: + for t in results[stem]["tests"]: + comp_data[comp]["rows"].append( + {"name": t["name"], "outcome": t["outcome"], "disabled": disabled} + ) + elif not disabled: + # Report expected but missing — test was not run + comp_data[comp]["rows"].append( + {"name": f"[{stem} — report missing]", "outcome": "not_run", "disabled": False} + ) + + return comp_data + + +def build_ci_section(comp_data, generated_date, l2_passed, l2_collected, l2_failed, + feature_scenarios, gap_to_100): + """Return the full markdown string to insert between the CI markers.""" + lines = [] + + # ---- header ----------------------------------------------------------- + lines.append(f"*Generated: {generated_date} | L2 passed: {l2_passed}/{l2_collected}, " + f"failed: {l2_failed} | feature scenarios: {feature_scenarios}*") + lines.append("") + + # ---- per-component table ---------------------------------------------- + lines.append("| Component | Source Functions Exercised | Tests | Passed | Failed | Coverage |") + lines.append("|---|---|---:|---:|---:|---:|") + + grand_active = 0 + grand_passed = 0 + grand_failed = 0 + + for comp_label in COMPONENT_ORDER: + data = comp_data[comp_label] + rows = data["rows"] + src_str = ", ".join(f"`{fn}`" for fn in sorted(data["source_fns"])) or "—" + + # Split active vs disabled + active_rows = [r for r in rows if not r["disabled"]] + disabled_rows = [r for r in rows if r["disabled"]] + + if not active_rows and disabled_rows: + # Whole component is disabled + lines.append( + f"| {comp_label} | {src_str} | {len(disabled_rows)} " + "| DISABLED | DISABLED | *pending open-source* |" + ) + continue + + if not active_rows: + lines.append(f"| {comp_label} | {src_str} | 0 | — | — | — |") + continue + + n_total = len(active_rows) + n_passed = sum(1 for r in active_rows if r["outcome"] == "passed") + n_failed = sum(1 for r in active_rows if r["outcome"] in ("failed", "error")) + coverage = f"{n_passed / n_total * 100:.0f}%" if n_total else "—" + + grand_active += n_total + grand_passed += n_passed + grand_failed += n_failed + + lines.append( + f"| {comp_label} | {src_str} | {n_total} | {n_passed} | {n_failed} | {coverage} |" + ) + + overall = f"{grand_passed / grand_active * 100:.2f}%" if grand_active else "0%" + lines.append( + f"| **TOTAL (active)** | | **{grand_active}** " + f"| **{grand_passed}** | **{grand_failed}** | **{overall}** |" + ) + lines.append("") + lines.append(f"**Gap to 100% functional coverage: {gap_to_100}%**") + lines.append("") + + # ---- per-test detail table ------------------------------------------- + lines.append("
") + lines.append("Per-test function results (click to expand)") + lines.append("") + lines.append("| Component | Test Function | Outcome |") + lines.append("|---|---|:---:|") + + for comp_label in COMPONENT_ORDER: + data = comp_data[comp_label] + for row in data["rows"]: + icon = OUTCOME_ICON.get(row["outcome"], row["outcome"].upper()) + dis = " *(disabled)*" if row["disabled"] else "" + lines.append(f"| {comp_label} | `{row['name']}`{dis} | {icon} |") + + lines.append("") + lines.append("
") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Document updaters +# --------------------------------------------------------------------------- + +def update_l2_report(doc_path, comp_data, generated_date, l2_env): + """Replace the CI-generated section in L2_Analysis_Report.md.""" + if not os.path.isfile(doc_path): + print(f"ERROR: {doc_path} not found", file=sys.stderr) + return False + + l2_passed = l2_env.get("L2_PASSED", "?") + l2_collected = l2_env.get("L2_COLLECTED", "?") + l2_failed = l2_env.get("L2_FAILED", "?") + feature_scenarios = l2_env.get("FEATURE_SCENARIOS", "?") + gap_to_100 = l2_env.get("GAP_TO_100_PCT", "?") + + ci_block = build_ci_section( + comp_data, generated_date, + l2_passed, l2_collected, l2_failed, + feature_scenarios, gap_to_100, + ) + + with open(doc_path, "r", encoding="utf-8") as fh: + content = fh.read() + + START_MARKER = "" + END_MARKER = "" + + if START_MARKER not in content or END_MARKER not in content: + print( + f"ERROR: CI markers not found in {doc_path}. " + "Add and to the document.", + file=sys.stderr, + ) + return False + + new_content = re.sub( + re.escape(START_MARKER) + r".*?" + re.escape(END_MARKER), + START_MARKER + "\n" + ci_block + "\n" + END_MARKER, + content, + count=1, + flags=re.DOTALL, + ) + + with open(doc_path, "w", encoding="utf-8") as fh: + fh.write(new_content) + print(f"Updated: {doc_path}") + return True + + +def update_l1_report(doc_path, generated_date, l1_env): + """Replace the header section of L1_Analysis_Report.md (before ## Directory Breakdown).""" + if not os.path.isfile(doc_path): + print(f"ERROR: {doc_path} not found", file=sys.stderr) + return False + + line_pct = l1_env.get("LINE_PCT", "?") + line_hit = l1_env.get("LINE_HIT", "?") + line_total = l1_env.get("LINE_TOTAL", "?") + func_pct = l1_env.get("FUNC_PCT", "?") + func_hit = l1_env.get("FUNC_HIT", "?") + func_total = l1_env.get("FUNC_TOTAL", "?") + + new_header = ( + "# L1 Code Coverage Report\n\n" + "**Test File:** coverage.info \n" + f"**Date:** {generated_date} \n\n" + "## Summary\n" + f"- **Line Coverage:** {line_hit} / {line_total} (**{line_pct}%**)\n" + f"- **Function Coverage:** {func_hit} / {func_total} (**{func_pct}%**)\n\n" + ) + + with open(doc_path, "r", encoding="utf-8") as fh: + content = fh.read() + + # Replace everything before ## Directory Breakdown + new_content = re.sub( + r"\A.*?(?=^## Directory Breakdown)", + new_header, + content, + count=1, + flags=re.DOTALL | re.MULTILINE, + ) + if new_content == content: + new_content = re.sub( + r"\A.*?(?=^---)", + new_header, + content, + count=1, + flags=re.DOTALL | re.MULTILINE, + ) + if new_content == content: + new_content = new_header + content + + with open(doc_path, "w", encoding="utf-8") as fh: + fh.write(new_content) + print(f"Updated: {doc_path}") + return True + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="Update L1/L2 coverage markdown docs.") + parser.add_argument("--reports-dir", default="ci-artifacts/l2-reports", + help="Directory containing pytest JSON report files") + parser.add_argument("--l1-env", default="ci-artifacts/l1/l1_metrics.env", + help="Path to L1 metrics env file") + parser.add_argument("--l2-env", default="ci-artifacts/l2/l2_metrics.env", + help="Path to L2 metrics env file") + parser.add_argument("--l1-doc", default="test/docs/L1_Analysis_Report.md") + parser.add_argument("--l2-doc", default="test/docs/L2_Analysis_Report.md") + parser.add_argument("--date", default=None, + help="Override generated date (YYYY-MM-DD). Defaults to today UTC.") + args = parser.parse_args() + + from datetime import datetime, timezone + generated_date = args.date or datetime.now(timezone.utc).strftime("%Y-%m-%d") + + l1_env = load_env_file(args.l1_env) + l2_env = load_env_file(args.l2_env) + + reports = load_reports(args.reports_dir) + comp_data = compute_component_metrics(reports) + + ok_l1 = update_l1_report(args.l1_doc, generated_date, l1_env) + ok_l2 = update_l2_report(args.l2_doc, comp_data, generated_date, l2_env) + + if not ok_l1 or not ok_l2: + sys.exit(1) + + +if __name__ == "__main__": + main() From c05ed4372a27b670453cb5439133dcdae8fc5982 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 15:04:20 -0400 Subject: [PATCH 23/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 1f3eff43..3ebe9781 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -207,13 +207,13 @@ jobs: # Extract the CI-generated component table from the updated doc COMPONENT_TABLE=$(python3 - <<'PY' -import re, sys -with open("test/docs/L2_Analysis_Report.md", "r") as f: - content = f.read() -m = re.search(r"\n(.*?)\n", content, re.DOTALL) -print(m.group(1) if m else "_Component table not available_") -PY -) + import re, sys + with open("test/docs/L2_Analysis_Report.md", "r") as f: + content = f.read() + m = re.search(r"\n(.*?)\n", content, re.DOTALL) + print(m.group(1) if m else "_Component table not available_") + PY + ) cat > /tmp/pr_coverage_comment.md << EOF From be19a2e84945a54e9e5e6a25b12b8100fdbdfa51 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 15:06:42 -0400 Subject: [PATCH 24/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 3ebe9781..54ae7dd6 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -216,21 +216,21 @@ jobs: ) cat > /tmp/pr_coverage_comment.md << EOF - -## PR Coverage Summary + + ## PR Coverage Summary -### L1 Code Coverage -| Metric | Value | -|---|---:| -| Line coverage | ${LINE_PCT}% (${LINE_HIT} / ${LINE_TOTAL}) | -| Function coverage | ${FUNC_PCT}% (${FUNC_HIT} / ${FUNC_TOTAL}) | + ### L1 Code Coverage + | Metric | Value | + |---|---:| + | Line coverage | ${LINE_PCT}% (${LINE_HIT} / ${LINE_TOTAL}) | + | Function coverage | ${FUNC_PCT}% (${FUNC_HIT} / ${FUNC_TOTAL}) | -### L2 Functional Coverage by Component -${COMPONENT_TABLE} + ### L2 Functional Coverage by Component + ${COMPONENT_TABLE} -Coverage docs status: **${DOC_STATUS}** -Updated: \`test/docs/L1_Analysis_Report.md\` · \`test/docs/L2_Analysis_Report.md\` -EOF + Coverage docs status: **${DOC_STATUS}** + Updated: \`test/docs/L1_Analysis_Report.md\` · \`test/docs/L2_Analysis_Report.md\` + EOF cat /tmp/pr_coverage_comment.md >> "$GITHUB_STEP_SUMMARY" From da55b5d8ce27860f079e767feb9efb0b9a0c7a58 Mon Sep 17 00:00:00 2001 From: nhanasi Date: Fri, 24 Jul 2026 15:14:46 -0400 Subject: [PATCH 25/25] Update L1_L2_CoverageReport.yml --- .github/workflows/L1_L2_CoverageReport.yml | 29 ++++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/L1_L2_CoverageReport.yml b/.github/workflows/L1_L2_CoverageReport.yml index 54ae7dd6..35fda8e9 100644 --- a/.github/workflows/L1_L2_CoverageReport.yml +++ b/.github/workflows/L1_L2_CoverageReport.yml @@ -215,22 +215,19 @@ jobs: PY ) - cat > /tmp/pr_coverage_comment.md << EOF - - ## PR Coverage Summary - - ### L1 Code Coverage - | Metric | Value | - |---|---:| - | Line coverage | ${LINE_PCT}% (${LINE_HIT} / ${LINE_TOTAL}) | - | Function coverage | ${FUNC_PCT}% (${FUNC_HIT} / ${FUNC_TOTAL}) | - - ### L2 Functional Coverage by Component - ${COMPONENT_TABLE} - - Coverage docs status: **${DOC_STATUS}** - Updated: \`test/docs/L1_Analysis_Report.md\` · \`test/docs/L2_Analysis_Report.md\` - EOF + { + printf '\n' + printf '## PR Coverage Summary\n\n' + printf '### L1 Code Coverage\n' + printf '| Metric | Value |\n' + printf '|---|---:|\n' + printf '| Line coverage | %s%%%% (%s / %s) |\n' "${LINE_PCT}" "${LINE_HIT}" "${LINE_TOTAL}" + printf '| Function coverage | %s%%%% (%s / %s) |\n' "${FUNC_PCT}" "${FUNC_HIT}" "${FUNC_TOTAL}" + printf '\n### L2 Functional Coverage by Component\n' + printf '%s\n' "${COMPONENT_TABLE}" + printf '\nCoverage docs status: **%s**\n' "${DOC_STATUS}" + printf 'Updated: `test/docs/L1_Analysis_Report.md` . `test/docs/L2_Analysis_Report.md`\n' + } > /tmp/pr_coverage_comment.md cat /tmp/pr_coverage_comment.md >> "$GITHUB_STEP_SUMMARY"