diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index a06430a85..62bfa58c8 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -2,7 +2,12 @@ name: L2 Integration Tests on: pull_request: - branches: [ develop ] + branches: [ feature/l2docs ] + push: + branches: [ feature/l2docs ] + +permissions: + contents: write env: AUTOMATICS_UNAME: ${{ secrets.AUTOMATICS_UNAME }} @@ -18,6 +23,7 @@ jobs: uses: actions/checkout@v4 with: path: remote_debugger + token: ${{ secrets.GITHUB_TOKEN }} - name: Check out dependent repostiories uses: actions/checkout@v4 @@ -88,3 +94,28 @@ jobs: run: | docker cp /tmp/L2_TEST_RESULTS ci-container:/tmp/L2_TEST_RESULTS docker exec -i ci-container bash -c "echo 'Contents in workspace directory' && ls -l && echo '===============================' && echo 'Contents in /tmp/L2_TEST_RESULTS' && ls -l /tmp/L2_TEST_RESULTS && echo '===============================' && git config --global --add safe.directory /mnt/L2_CONTAINER_SHARED_VOLUME/remote_debugger && gtest-json-result-push.py /tmp/L2_TEST_RESULTS https://rdkeorchestrationservice.apps.cloud.comcast.net/rdke_orchestration_api/push_unit_test_results /mnt/L2_CONTAINER_SHARED_VOLUME/remote_debugger" + + - name: Copy lcov tracefile from container + id: copy_tracefile + if: github.event_name == 'push' + continue-on-error: true + run: docker cp native-platform:/tmp/l2_coverage/coverage.info /tmp/coverage.info + + - name: Generate L2_Coverage.md + if: github.event_name == 'push' && steps.copy_tracefile.outcome == 'success' + run: | + python3 remote_debugger/test/functional-tests/generate_l2_coverage_report.py \ + --tracefile /tmp/coverage.info \ + --features-dir remote_debugger/test/functional-tests/features \ + --tests-dir remote_debugger/test/functional-tests/tests \ + --output remote_debugger/test/functional-tests/L2_Coverage.md + + - name: Commit L2_Coverage.md + if: github.event_name == 'push' && steps.copy_tracefile.outcome == 'success' + run: | + cd remote_debugger + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add test/functional-tests/L2_Coverage.md + git diff --cached --quiet || git commit -m "ci: regenerate L2_Coverage.md [skip ci]" + git push diff --git a/cov_build.sh b/cov_build.sh index 79c930133..c0871e32a 100644 --- a/cov_build.sh +++ b/cov_build.sh @@ -86,5 +86,5 @@ cd $WORKDIR autoreconf -i autoupdate ./configure --prefix=${INSTALL_DIR} --enable-iarmbusSupport=yes -make remotedebugger_CFLAGS="-I/usr/include/cjson -I/usr/local/include/wdmp-c -I/usr/local/include/rbus -I/usr/local/include -I./unittest/mocks -I/usr/local/include/trower-base64 -DIARMBUS_SUPPORT -DUSECOV -DUSE_L2_SUPPORT" remotedebugger_LDFLAGS="-L/usr/local/lib -lrdkloggers -lcjson -lrfcapi -lrbus -lmsgpackc -lsecure_wrapper -lwebconfig_framework -lIARMBus -ltr181api -L/usr/local/lib/x86_64-linux-gnu -ltrower-base64 -L/usr/lib/x86_64-linux-gnu" +make remotedebugger_CFLAGS="-I/usr/include/cjson -I/usr/local/include/wdmp-c -I/usr/local/include/rbus -I/usr/local/include -I./unittest/mocks -I/usr/local/include/trower-base64 -DIARMBUS_SUPPORT -DUSECOV -DUSE_L2_SUPPORT --coverage" remotedebugger_LDFLAGS="-L/usr/local/lib -lrdkloggers -lcjson -lrfcapi -lrbus -lmsgpackc -lsecure_wrapper -lwebconfig_framework -lIARMBus -ltr181api -L/usr/local/lib/x86_64-linux-gnu -ltrower-base64 -L/usr/lib/x86_64-linux-gnu --coverage" make install diff --git a/run_l2.sh b/run_l2.sh index 7caeb2608..adc1f1be4 100644 --- a/run_l2.sh +++ b/run_l2.sh @@ -22,19 +22,23 @@ RESULT_DIR="/tmp/l2_test_report" STATIC_PROFILE_DIR="/etc/rrd" OUTPUT_DIR="/tmp/rrd" LIB_DIR="/lib/rdk" +COV_DIR="/tmp/l2_coverage" + +export RRD_COVERAGE_MODE=1 mkdir -p "$RESULT_DIR" mkdir -p "$OUTPUT_DIR" mkdir -p "$STATIC_PROFILE_DIR" mkdir -p "$LIB_DIR" +mkdir -p "$COV_DIR" mkdir -p /media/apps/RDK-RRD-Test/etc/rrd touch /media/apps/RDK-RRD-Test/etc/rrd/remote_debugger.json echo "AA:BB:CC:DD:EE:FF" >> /tmp/.estb_mac -apt-get remove systemd -apt-get update && apt-get install -y tcpdump +apt-get remove -y systemd || true +apt-get update && apt-get install -y tcpdump lcov echo "LOG_PATH=/opt/logs" >> /etc/include.properties cp remote_debugger.json /etc/rrd/remote_debugger.json @@ -60,6 +64,11 @@ ln -s /usr/local/bin/journalctl /usr/bin/journalctl rm -rf /tmp/rrd/* rm -rf /opt/logs/remotedebugger.log* +# lcov baseline — capture zero counters before any test runs +lcov --zerocounters --directory "$(pwd)/src" +lcov --capture --initial --directory "$(pwd)/src" \ + --output-file "$COV_DIR/coverage_base.info" --rc lcov_branch_coverage=1 + # Run L2 Test cases pytest --json-report --json-report-summary --json-report-file $RESULT_DIR/rrd_dynamic_profile_missing_report.json test/functional-tests/tests/test_rrd_dynamic_profile_missing_report.py pytest --json-report --json-report-summary --json-report-file $RESULT_DIR/test_category.json test/functional-tests/tests/test_rrd_dynamic_subcategory_report.py @@ -90,5 +99,18 @@ pytest --json-report --json-report-summary --json-report-file $RESULT_DIR/rrd_pr pytest --json-report --json-report-summary --json-report-file $RESULT_DIR/rrd_dynamic_profile_rdm_node_length_exceeded.json test/functional-tests/tests/test_rrd_dynamic_profile_rdm_node_length_exceeded.py - +# Capture, filter, and report coverage +lcov --capture --directory "$(pwd)/src" \ + --output-file "$COV_DIR/coverage_test.info" --rc lcov_branch_coverage=1 +lcov --add-tracefile "$COV_DIR/coverage_base.info" \ + --add-tracefile "$COV_DIR/coverage_test.info" \ + --output-file "$COV_DIR/coverage_merged.info" --rc lcov_branch_coverage=1 +lcov --remove "$COV_DIR/coverage_merged.info" '/usr/*' \ + --output-file "$COV_DIR/coverage.info" --rc lcov_branch_coverage=1 +genhtml "$COV_DIR/coverage.info" \ + --output-directory "$COV_DIR/html" \ + --title "Remote Debugger L2 Coverage" \ + --branch-coverage --legend +echo "Coverage report : $COV_DIR/html/index.html" +echo "lcov tracefile : $COV_DIR/coverage.info" diff --git a/src/rrdMain.c b/src/rrdMain.c index 40fdc9bc5..ede604b96 100644 --- a/src/rrdMain.c +++ b/src/rrdMain.c @@ -23,7 +23,18 @@ #include "rrdDynamic.h" #include "rrdEventProcess.h" #include "rrdInterface.h" - +#ifdef USECOV +#include +#include +extern void __gcov_dump(void); +/* Flush gcov counters on SIGTERM so lcov captures L2 exercise data. */ +static void rrd_gcov_sigterm_handler(int sig) +{ + (void)sig; + __gcov_dump(); + _exit(0); +} +#endif devicePropertiesData devPropData; @@ -139,6 +150,9 @@ int main(int argc, char *argv[]) pthread_t RRDTR69ThreadID; rdk_logger_init(DEBUG_INI_FILE); +#ifdef USECOV + signal(SIGTERM, rrd_gcov_sigterm_handler); +#endif #if !defined(GTEST_ENABLE) /* Store Device Info.*/ RRDStoreDeviceInfo(&devPropData); diff --git a/test/functional-tests/L2_Coverage.md b/test/functional-tests/L2_Coverage.md new file mode 100644 index 000000000..045d2d4a5 --- /dev/null +++ b/test/functional-tests/L2_Coverage.md @@ -0,0 +1,284 @@ +# Remote Debugger L2 Coverage Report + +**Generated:** 2026-08-14 +**Component:** `remotedebugger` (`src/`) +**Test suite:** `test/functional-tests/` +**Coverage tool:** lcov (source-level instrumentation via `--coverage`) + +--- + +## 1. Executive Summary + +| Metric | Value | +|---|:---:| +| Feature files | 22 | +| Feature scenarios | 103 | +| Test files (pytest) | 23 | +| Test functions (`test_*`) | 118 | +| Feature → Test mapped pairs | 19 / 22 (+4 orphan tests) | +| **Line coverage (lcov)** | **49.4%** (1487 of 3012 lines) | +| **Branch coverage (lcov)** | **36.8%** (503 of 1365 branches) | +| **Function coverage (lcov)** | **60.3%** (76 of 126 functions) | + +--- + +## 2. Feature ↔ Test Mapping + +### 2.1 Mapped Pairs + +| # | Feature File | Scenarios | Test File | Tests | Gap | +|:---:|---|:---:|---|:---:|:---:| +| 1 | `rrd_append_report.feature` | 4 | `test_rrd_append_report.py` | 7 | — | +| 2 | `rrd_background_cmd_static_profile_report.feature` | 5 | `test_rrd_background_cmd_static_profile_report.py` | 5 | — | +| 3 | `rrd_c_api_upload.feature` | 21 | `test_rrd_c_api_upload.py` | 5 | **16 missing** | +| 4 | `rrd_corrupted_static_profile_report.feature` | 4 | `test_rrd_corrupted_static_profile_report.py` | 4 | — | +| 5 | `rrd_debug_report_upload.feature` | 6 | `test_rrd_debug_report_upload.py` | 6 | — | +| 6 | `rrd_deepsleep_static_report.feature` | 2 | `test_rrd_deepsleep_static_report.py` | 5 | — | +| 7 | `rrd_dynamic_profile_missing_report.feature` | 4 | `test_rrd_dynamic_profile_missing_report.py` | 7 | — | +| 8 | `rrd_dynamic_profile_report.feature` | 5 | `test_rrd_dynamic_profile_report.py` | 9 | — | +| 9 | `rrd_empty_issuetype_event.feature` | 2 | `test_rrd_empty_issuetype_event.py` | 2 | — | +| 10 | `rrd_harmful_command_static_report.feature` | 5 | `test_rrd_harmful_command_static_report.py` | 5 | — | +| 11 | `rrd_single_instance.feature` | 1 | `test_rrd_single_instance.py` | 3 | — | +| 12 | `rrd_start_control.feature` | 2 | `test_rrd_start_control.py` | 1 | **1 missing** | +| 13 | `rrd_start_subscribe_and_wait.feature` | 1 | `test_rrd_start_subscribe_and_wait.py` | 4 | — | +| 14 | `rrd_static_profile_category_report.feature` | 5 | `test_rrd_static_profile_category_report.py` | 5 | — | +| 15 | `rrd_static_profile_missing_command_report.feature` | 5 | `test_rrd_static_profile_missing_command_report.py` | 5 | — | +| 16 | `rrd_static_profile_report.feature` | 5 | `test_rrd_static_profile_report.py` | 5 | — | +| 17 | `test_rrd_dynamic_profile_harmful_report.feature` | 5 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | — | +| 18 | `test_rrd_static_profile_report_with_suffix.feature` | 4 | `test_rrd_static_profile_report_with_suffix.py` | 5 | — | +| 19 | `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | — | +| | **Totals** | **90** | | **95** | | + +### 2.2 Orphan Tests (test exists, no feature file) + +| Test File | Tests | Note | +|---|:---:|---| +| `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | **Missing `.feature` file** | +| `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | **Missing `.feature` file** | +| `test_rrd_dynamic_subcategory_report.py` | 7 | **Missing `.feature` file** | +| `test_rrd_profile_data.py` | 3 | **Missing `.feature` file** | + +### 2.3 Orphan Features (feature exists, no test file) + +| Feature File | Scenarios | Note | +|---|:---:|---| +| `rrd_append_dynamic_profile_static_not_found.feature` | 4 | **Missing test file** | +| `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | **Missing test file** | +| `rrd_dynamic_profile_subcategory_report.feature` | 5 | **Missing test file** | + +--- + +## 3. Source Module Coverage (lcov) + +| Module | Lines | Functions | Branches | Coverage Bar | +|---|:---:|:---:|:---:|---| +| `rrdMain.c` | 60.3% (38/63) | 100.0% (4/4) | 42.9% (6/14) | `████████████░░░░░░░░` | +| `rrdInterface.c` | 42.4% (181/427) | 59.1% (13/22) | 33.3% (56/168) | `████████░░░░░░░░░░░░` | +| `rrdEventProcess.c` | 69.3% (232/335) | 81.8% (9/11) | 48.6% (68/140) | `██████████████░░░░░░` | +| `rrdJsonParser.c` | 81.6% (400/490) | 86.7% (13/15) | 66.0% (128/194) | `████████████████░░░░` | +| `rrdRunCmdThread.c` | 55.2% (117/212) | 60.0% (6/10) | 34.5% (20/58) | `███████████░░░░░░░░░` | +| `rrdCommandSanity.c` | 93.1% (67/72) | 100.0% (3/3) | 71.9% (23/32) | `███████████████████░` | +| `rrdDynamic.c` | 40.4% (59/146) | 40.0% (2/5) | 24.6% (14/57) | `████████░░░░░░░░░░░░` | +| `rrdExecuteScript.c` | 94.1% (16/17) | 100.0% (2/2) | 75.0% (6/8) | `███████████████████░` | +| `rrdMsgPackDecoder.c` | 0.0% (0/280) | 0.0% (0/15) | 0.0% (0/138) | `░░░░░░░░░░░░░░░░░░░░` | +| `rrd_config.c` | 42.8% (95/222) | 66.7% (6/9) | 27.6% (53/192) | `█████████░░░░░░░░░░░` | +| `rrd_sysinfo.c` | 32.1% (36/112) | 33.3% (2/6) | 19.6% (11/56) | `██████░░░░░░░░░░░░░░` | +| `rrd_logproc.c` | 48.6% (36/74) | 75.0% (3/4) | 47.9% (23/48) | `██████████░░░░░░░░░░` | +| `rrd_archive.c` | 47.2% (126/267) | 80.0% (8/10) | 46.5% (66/142) | `█████████░░░░░░░░░░░` | +| `rrd_upload.c` | 40.7% (37/91) | 60.0% (3/5) | 30.4% (14/46) | `████████░░░░░░░░░░░░` | +| `rrdIarmEvents.c` | 8.6% (12/140) | 25.0% (1/4) | 8.3% (4/48) | `██░░░░░░░░░░░░░░░░░░` | +| `uploadRRDLogs.c` | 54.7% (35/64) | 100.0% (1/1) | 45.8% (11/24) | `███████████░░░░░░░░░` | + +--- + +## 4. Per-Behavior Coverage Detail + +> Legend — **YES**: tested by an L2 scenario | **NO**: no test exists | **PARTIAL**: subset covered + +### 4.1 Daemon Lifecycle + +| Behavior | Covered | +|---|:---:| +| RBUS subscription + event wait | YES | +| RFC enable → daemon starts | YES | +| RFC disable → daemon stops | YES | +| Single instance enforcement | YES | +| Message queue creation failure | NO | +| Event thread creation failure | NO | +| Signal handling / graceful shutdown | NO | +| Device info file read failure | NO | + +### 4.2 Static Profile Processing + +| Behavior | Covered | +|---|:---:| +| Config file exists check | YES | +| IssueType event trigger + message flow | YES | +| JSON parse success + command execution | YES | +| Upload report success / failure | YES | +| Category-only issue type | YES | +| Suffixed issue type | YES | +| Overlength suffix (negative) | YES | +| Background command execution | YES | +| Missing command in profile | YES | +| Corrupted / invalid JSON profile | YES | + +### 4.3 Dynamic Profile Processing + +| Behavior | Covered | +|---|:---:| +| Dynamic profile fallback (static miss) | YES | +| Dynamic subcategory | YES | +| Dynamic profile missing → RDM trigger | YES | +| Append mode (static + dynamic) | YES | +| Append when static not found | YES | +| RDM download event (cache miss) | NO | +| Dynamic profile JSON parse failure | NO | + +### 4.4 Harmful Command Detection + +| Behavior | Covered | +|---|:---:| +| Static profile harmful command abort | YES | +| Dynamic profile harmful command abort | YES | +| Macro replacement edge cases | NO | +| Background command modification | PARTIAL | + +### 4.5 Event Handling + +| Behavior | Covered | +|---|:---:| +| IssueType RBUS event | YES | +| Empty IssueType event | YES | +| Deep sleep event | YES | +| WebCfg event (MsgPack decode) | NO | +| WebCfg corrupted data | NO | +| Multiple simultaneous IssueType events | NO | +| Invalid deep sleep event type | NO | + +### 4.6 Upload & Archive + +| Behavior | Covered | +|---|:---:| +| Upload via shell script | YES | +| Upload + download validation | YES | +| C API `rrd_upload_orchestrate` (happy path) | YES | +| C API NULL parameters | NO | +| C API empty / non-existent directory | NO | +| C API config loading / MAC retrieval | NO | +| C API archive creation + cleanup | NO | +| Concurrent upload lock | NO | +| Archive CPU throttle | NO | + + +--- + +## 5. Scenario-to-Test Gap Analysis + +| Feature File | Scenarios | Tests | Missing | +|---|:---:|:---:|:---:| +| `rrd_c_api_upload.feature` | 21 | 5 | **16** | +| `rrd_start_control.feature` | 2 | 1 | **1** | + +**Orphan tests** (no feature file — behavior is tested but not documented): + +- `test_rrd_append_dynamic_profile_static_notfound.py` +- `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` +- `test_rrd_dynamic_subcategory_report.py` +- `test_rrd_profile_data.py` + +--- + +## 6. Gap Recommendations + +### Priority 1 — Must Fix + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 1 | Implement missing `test_rrd_c_api_upload.py` scenarios (16 of 21 unimplemented) | `rrd_upload.c` | +| 2 | Add `rrd_profile_data.feature` for the existing `test_rrd_profile_data.py` | `rrdInterface.c` | +| 3 | Add WebCfg / MsgPack event L2 test (`rrd_webcfg_event.feature` + test) | `rrdMsgPackDecoder.c`, `rrdEventProcess.c` | + +### Priority 2 — Should Fix + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 4 | Upload lock contention test | `rrd_upload.c` | +| 5 | Configuration fallback chain (RFC → DCM → dcm.properties) | `rrd_config.c` | +| 6 | Archive CPU throttle logic | `rrd_archive.c` | +| 7 | RDM download event with dynamic-profile cache miss | `rrdDynamic.c`, `rrdInterface.c` | +| 8 | Dynamic profile JSON parse failure | `rrdDynamic.c` | + +### Priority 3 — Nice to Have + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 9 | RBUS registration / unregistration failure injection | `rrdInterface.c` | +| 10 | Message queue creation failure | `rrdMain.c` | +| 11 | Event thread creation failure | `rrdMain.c` | +| 12 | Directory creation / chdir failures | `rrdJsonParser.c` | +| 13 | `systemd-run` / `journalctl` execution failures | `rrdRunCmdThread.c` | +| 14 | Output file write errors | `rrdRunCmdThread.c` | +| 15 | Invalid deep sleep event type | `rrdDynamic.c` | + + +--- + +## 7. Appendix: File Inventory + +### Feature Files + +| # | File | Scenarios | +|:---:|---|:---:| +| 1 | `rrd_append_dynamic_profile_static_not_found.feature` | 4 | +| 2 | `rrd_append_report.feature` | 4 | +| 3 | `rrd_background_cmd_static_profile_report.feature` | 5 | +| 4 | `rrd_c_api_upload.feature` | 21 | +| 5 | `rrd_corrupted_static_profile_report.feature` | 4 | +| 6 | `rrd_debug_report_upload.feature` | 6 | +| 7 | `rrd_deepsleep_static_report.feature` | 2 | +| 8 | `rrd_dynamic_profile_missing_report.feature` | 4 | +| 9 | `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | +| 10 | `rrd_dynamic_profile_report.feature` | 5 | +| 11 | `rrd_dynamic_profile_subcategory_report.feature` | 5 | +| 12 | `rrd_empty_issuetype_event.feature` | 2 | +| 13 | `rrd_harmful_command_static_report.feature` | 5 | +| 14 | `rrd_single_instance.feature` | 1 | +| 15 | `rrd_start_control.feature` | 2 | +| 16 | `rrd_start_subscribe_and_wait.feature` | 1 | +| 17 | `rrd_static_profile_category_report.feature` | 5 | +| 18 | `rrd_static_profile_missing_command_report.feature` | 5 | +| 19 | `rrd_static_profile_report.feature` | 5 | +| 20 | `test_rrd_dynamic_profile_harmful_report.feature` | 5 | +| 21 | `test_rrd_static_profile_report_with_suffix.feature` | 4 | +| 22 | `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | +| | **Total** | **103** | + +### Test Files + +| # | File | Tests | +|:---:|---|:---:| +| 1 | `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | +| 2 | `test_rrd_append_report.py` | 7 | +| 3 | `test_rrd_background_cmd_static_profile_report.py` | 5 | +| 4 | `test_rrd_c_api_upload.py` | 5 | +| 5 | `test_rrd_corrupted_static_profile_report.py` | 4 | +| 6 | `test_rrd_debug_report_upload.py` | 6 | +| 7 | `test_rrd_deepsleep_static_report.py` | 5 | +| 8 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | +| 9 | `test_rrd_dynamic_profile_missing_report.py` | 7 | +| 10 | `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | +| 11 | `test_rrd_dynamic_profile_report.py` | 9 | +| 12 | `test_rrd_dynamic_subcategory_report.py` | 7 | +| 13 | `test_rrd_empty_issuetype_event.py` | 2 | +| 14 | `test_rrd_harmful_command_static_report.py` | 5 | +| 15 | `test_rrd_profile_data.py` | 3 | +| 16 | `test_rrd_single_instance.py` | 3 | +| 17 | `test_rrd_start_control.py` | 1 | +| 18 | `test_rrd_start_subscribe_and_wait.py` | 4 | +| 19 | `test_rrd_static_profile_category_report.py` | 5 | +| 20 | `test_rrd_static_profile_missing_command_report.py` | 5 | +| 21 | `test_rrd_static_profile_report.py` | 5 | +| 22 | `test_rrd_static_profile_report_with_suffix.py` | 5 | +| 23 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | +| | **Total** | **118** | diff --git a/test/functional-tests/generate_l2_coverage_report.py b/test/functional-tests/generate_l2_coverage_report.py new file mode 100644 index 000000000..3e16c9e99 --- /dev/null +++ b/test/functional-tests/generate_l2_coverage_report.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +""" +Generate test/functional-tests/L2_Coverage.md. + +Inputs +------ + --tracefile lcov .info file produced by run_l2.sh + --features-dir directory containing .feature files + --tests-dir directory containing test_*.py files + --output path for the generated markdown file + +Usage +----- + python3 generate_l2_coverage_report.py \ + --tracefile /tmp/l2_coverage/coverage.info \ + --features-dir test/functional-tests/features \ + --tests-dir test/functional-tests/tests \ + --output test/functional-tests/L2_Coverage.md +""" + +import argparse +import datetime +import os +import re +import sys + + +# --------------------------------------------------------------------------- +# lcov tracefile parser +# --------------------------------------------------------------------------- + +def parse_tracefile(path): + """Return (per_file_dict, summary_dict) from an lcov .info file.""" + files = {} + cur = None + with open(path) as fh: + for raw in fh: + line = raw.rstrip() + if line.startswith('SF:'): + name = os.path.basename(line[3:]) + cur = {'name': name, 'lh': 0, 'lf': 0, + 'fnh': 0, 'fnf': 0, 'brh': 0, 'brf': 0} + elif line == 'end_of_record' and cur: + files[cur['name']] = cur + cur = None + elif cur: + if line.startswith('LH:'): cur['lh'] = int(line[3:]) + elif line.startswith('LF:'): cur['lf'] = int(line[3:]) + elif line.startswith('FNH:'): cur['fnh'] = int(line[4:]) + elif line.startswith('FNF:'): cur['fnf'] = int(line[4:]) + elif line.startswith('BRH:'): cur['brh'] = int(line[4:]) + elif line.startswith('BRF:'): cur['brf'] = int(line[4:]) + + def tot(key): return sum(v[key] for v in files.values()) + summary = { + 'lines_pct': _pct(tot('lh'), tot('lf')), + 'lines_det': f"{tot('lh')} of {tot('lf')} lines", + 'funcs_pct': _pct(tot('fnh'), tot('fnf')), + 'funcs_det': f"{tot('fnh')} of {tot('fnf')} functions", + 'branches_pct': _pct(tot('brh'), tot('brf')), + 'branches_det': f"{tot('brh')} of {tot('brf')} branches", + } + return files, summary + + +def _pct(hit, found): + if found == 0: + return 'N/A' + return f'{hit / found * 100:.1f}%' + + +def _bar(hit, found, width=20): + if found == 0: + return '░' * width + filled = round(hit / found * width) + return '█' * filled + '░' * (width - filled) + + +# --------------------------------------------------------------------------- +# test directory scanners +# --------------------------------------------------------------------------- + +def scan_features(features_dir): + """Return {filename: scenario_count}.""" + result = {} + for fn in sorted(os.listdir(features_dir)): + if fn.endswith('.feature'): + result[fn] = _count_scenarios(os.path.join(features_dir, fn)) + return result + + +def scan_tests(tests_dir): + """Return {filename: test_function_count}.""" + result = {} + for fn in sorted(os.listdir(tests_dir)): + if fn.startswith('test_') and fn.endswith('.py'): + result[fn] = _count_test_funcs(os.path.join(tests_dir, fn)) + return result + + +def _count_scenarios(path): + count = 0 + with open(path, errors='replace') as fh: + for line in fh: + if re.match(r'\s*Scenario(\s+Outline)?:', line): + count += 1 + return count + + +def _count_test_funcs(path): + count = 0 + with open(path, errors='replace') as fh: + for line in fh: + if re.match(r'^def test_', line): + count += 1 + return count + + +# --------------------------------------------------------------------------- +# feature → test matching +# --------------------------------------------------------------------------- + +def build_mapping(features, tests): + """ + Return (pairs, orphan_features, orphan_tests). + pairs = list of (feature_file, scenarios, test_file_or_None, test_funcs_or_0) + """ + unmatched_tests = set(tests.keys()) + pairs = [] + for feat, scen in features.items(): + stem = feat.replace('.feature', '') + candidate = (stem if stem.startswith('test_') else 'test_' + stem) + '.py' + if candidate in tests: + pairs.append((feat, scen, candidate, tests[candidate])) + unmatched_tests.discard(candidate) + else: + pairs.append((feat, scen, None, 0)) + orphan_tests = {t: tests[t] for t in sorted(unmatched_tests)} + orphan_feats = [p[0] for p in pairs if p[2] is None] + return pairs, orphan_feats, orphan_tests + + +# --------------------------------------------------------------------------- +# static sections (priorities / recommendations — human judgment) +# --------------------------------------------------------------------------- + +_RECOMMENDATIONS = """\ +## 6. Gap Recommendations + +### Priority 1 — Must Fix + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 1 | Implement missing `test_rrd_c_api_upload.py` scenarios (16 of 21 unimplemented) | `rrd_upload.c` | +| 2 | Add `rrd_profile_data.feature` for the existing `test_rrd_profile_data.py` | `rrdInterface.c` | +| 3 | Add WebCfg / MsgPack event L2 test (`rrd_webcfg_event.feature` + test) | `rrdMsgPackDecoder.c`, `rrdEventProcess.c` | + +### Priority 2 — Should Fix + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 4 | Upload lock contention test | `rrd_upload.c` | +| 5 | Configuration fallback chain (RFC → DCM → dcm.properties) | `rrd_config.c` | +| 6 | Archive CPU throttle logic | `rrd_archive.c` | +| 7 | RDM download event with dynamic-profile cache miss | `rrdDynamic.c`, `rrdInterface.c` | +| 8 | Dynamic profile JSON parse failure | `rrdDynamic.c` | + +### Priority 3 — Nice to Have + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 9 | RBUS registration / unregistration failure injection | `rrdInterface.c` | +| 10 | Message queue creation failure | `rrdMain.c` | +| 11 | Event thread creation failure | `rrdMain.c` | +| 12 | Directory creation / chdir failures | `rrdJsonParser.c` | +| 13 | `systemd-run` / `journalctl` execution failures | `rrdRunCmdThread.c` | +| 14 | Output file write errors | `rrdRunCmdThread.c` | +| 15 | Invalid deep sleep event type | `rrdDynamic.c` | +""" + +_BEHAVIOR_DETAIL = """\ +## 4. Per-Behavior Coverage Detail + +> Legend — **YES**: tested by an L2 scenario | **NO**: no test exists | **PARTIAL**: subset covered + +### 4.1 Daemon Lifecycle + +| Behavior | Covered | +|---|:---:| +| RBUS subscription + event wait | YES | +| RFC enable → daemon starts | YES | +| RFC disable → daemon stops | YES | +| Single instance enforcement | YES | +| Message queue creation failure | NO | +| Event thread creation failure | NO | +| Signal handling / graceful shutdown | NO | +| Device info file read failure | NO | + +### 4.2 Static Profile Processing + +| Behavior | Covered | +|---|:---:| +| Config file exists check | YES | +| IssueType event trigger + message flow | YES | +| JSON parse success + command execution | YES | +| Upload report success / failure | YES | +| Category-only issue type | YES | +| Suffixed issue type | YES | +| Overlength suffix (negative) | YES | +| Background command execution | YES | +| Missing command in profile | YES | +| Corrupted / invalid JSON profile | YES | + +### 4.3 Dynamic Profile Processing + +| Behavior | Covered | +|---|:---:| +| Dynamic profile fallback (static miss) | YES | +| Dynamic subcategory | YES | +| Dynamic profile missing → RDM trigger | YES | +| Append mode (static + dynamic) | YES | +| Append when static not found | YES | +| RDM download event (cache miss) | NO | +| Dynamic profile JSON parse failure | NO | + +### 4.4 Harmful Command Detection + +| Behavior | Covered | +|---|:---:| +| Static profile harmful command abort | YES | +| Dynamic profile harmful command abort | YES | +| Macro replacement edge cases | NO | +| Background command modification | PARTIAL | + +### 4.5 Event Handling + +| Behavior | Covered | +|---|:---:| +| IssueType RBUS event | YES | +| Empty IssueType event | YES | +| Deep sleep event | YES | +| WebCfg event (MsgPack decode) | NO | +| WebCfg corrupted data | NO | +| Multiple simultaneous IssueType events | NO | +| Invalid deep sleep event type | NO | + +### 4.6 Upload & Archive + +| Behavior | Covered | +|---|:---:| +| Upload via shell script | YES | +| Upload + download validation | YES | +| C API `rrd_upload_orchestrate` (happy path) | YES | +| C API NULL parameters | NO | +| C API empty / non-existent directory | NO | +| C API config loading / MAC retrieval | NO | +| C API archive creation + cleanup | NO | +| Concurrent upload lock | NO | +| Archive CPU throttle | NO | +""" + + +# --------------------------------------------------------------------------- +# markdown builder +# --------------------------------------------------------------------------- + +def generate(pairs, orphan_feats, orphan_tests, files, summary, today): + total_scenarios = sum(p[1] for p in pairs) + total_test_funcs = sum(p[3] for p in pairs) + sum(orphan_tests.values()) + total_features = len(pairs) + total_tests = len(pairs) - len(orphan_feats) + len(orphan_tests) + mapped = sum(1 for p in pairs if p[2]) + gap_pairs = [p for p in pairs if p[2] and p[3] < p[1]] + + lines = [] + + # ── header ──────────────────────────────────────────────────────────────── + lines += [ + '# Remote Debugger L2 Coverage Report', + '', + f'**Generated:** {today} ', + '**Component:** `remotedebugger` (`src/`) ', + '**Test suite:** `test/functional-tests/` ', + '**Coverage tool:** lcov (source-level instrumentation via `--coverage`)', + '', + '---', + '', + ] + + # ── executive summary ───────────────────────────────────────────────────── + lines += [ + '## 1. Executive Summary', + '', + '| Metric | Value |', + '|---|:---:|', + f'| Feature files | {total_features} |', + f'| Feature scenarios | {total_scenarios} |', + f'| Test files (pytest) | {total_tests} |', + f'| Test functions (`test_*`) | {total_test_funcs} |', + f'| Feature → Test mapped pairs | {mapped} / {total_features}' + + (f' (+{len(orphan_tests)} orphan test{"s" if len(orphan_tests) != 1 else ""})' if orphan_tests else '') + ' |', + f'| **Line coverage (lcov)** | **{summary["lines_pct"]}** ({summary["lines_det"]}) |', + f'| **Branch coverage (lcov)** | **{summary["branches_pct"]}** ({summary["branches_det"]}) |', + f'| **Function coverage (lcov)** | **{summary["funcs_pct"]}** ({summary["funcs_det"]}) |', + '', + ] + + # ── feature ↔ test mapping ──────────────────────────────────────────────── + lines += [ + '---', + '', + '## 2. Feature ↔ Test Mapping', + '', + '### 2.1 Mapped Pairs', + '', + '| # | Feature File | Scenarios | Test File | Tests | Gap |', + '|:---:|---|:---:|---|:---:|:---:|', + ] + idx = 1 + for feat, scen, test, tfuncs in pairs: + if test is None: + continue + delta = tfuncs - scen + gap = '—' if delta >= 0 else f'**{abs(delta)} missing**' + lines.append(f'| {idx} | `{feat}` | {scen} | `{test}` | {tfuncs} | {gap} |') + idx += 1 + + total_scen_mapped = sum(p[1] for p in pairs if p[2]) + total_funcs_mapped = sum(p[3] for p in pairs if p[2]) + lines += [ + f'| | **Totals** | **{total_scen_mapped}** | | **{total_funcs_mapped}** | |', + '', + ] + + if orphan_tests: + lines += [ + '### 2.2 Orphan Tests (test exists, no feature file)', + '', + '| Test File | Tests | Note |', + '|---|:---:|---|', + ] + for tf, cnt in orphan_tests.items(): + lines.append(f'| `{tf}` | {cnt} | **Missing `.feature` file** |') + lines.append('') + + if orphan_feats: + lines += [ + '### 2.3 Orphan Features (feature exists, no test file)', + '', + '| Feature File | Scenarios | Note |', + '|---|:---:|---|', + ] + for ff in orphan_feats: + sc = features[ff] + lines.append(f'| `{ff}` | {sc} | **Missing test file** |') + lines.append('') + + # ── per-module lcov coverage ─────────────────────────────────────────────── + lines += [ + '---', + '', + '## 3. Source Module Coverage (lcov)', + '', + '| Module | Lines | Functions | Branches | Coverage Bar |', + '|---|:---:|:---:|:---:|---|', + ] + src_modules = [ + 'rrdMain.c', 'rrdInterface.c', 'rrdEventProcess.c', 'rrdJsonParser.c', + 'rrdRunCmdThread.c', 'rrdCommandSanity.c', 'rrdDynamic.c', + 'rrdExecuteScript.c', 'rrdMsgPackDecoder.c', + 'rrd_config.c', 'rrd_sysinfo.c', 'rrd_logproc.c', + 'rrd_archive.c', 'rrd_upload.c', 'rrdIarmEvents.c', 'uploadRRDLogs.c', + ] + for mod in src_modules: + if mod in files: + d = files[mod] + lp = _pct(d['lh'], d['lf']) + fp = _pct(d['fnh'], d['fnf']) + bp = _pct(d['brh'], d['brf']) + bar = _bar(d['lh'], d['lf']) + lines.append( + f'| `{mod}` | {lp} ({d["lh"]}/{d["lf"]}) ' + f'| {fp} ({d["fnh"]}/{d["fnf"]}) ' + f'| {bp} ({d["brh"]}/{d["brf"]}) ' + f'| `{bar}` |' + ) + else: + lines.append(f'| `{mod}` | — | — | — | no data |') + lines.append('') + + # ── per-behavior detail (static) ────────────────────────────────────────── + lines += ['---', '', _BEHAVIOR_DETAIL, ''] + + # ── gap analysis ────────────────────────────────────────────────────────── + lines += [ + '---', + '', + '## 5. Scenario-to-Test Gap Analysis', + '', + ] + if gap_pairs: + lines += [ + '| Feature File | Scenarios | Tests | Missing |', + '|---|:---:|:---:|:---:|', + ] + for feat, scen, test, tfuncs in gap_pairs: + lines.append(f'| `{feat}` | {scen} | {tfuncs} | **{scen - tfuncs}** |') + lines.append('') + else: + lines += ['All mapped feature files have sufficient test function coverage.', ''] + + if orphan_tests: + lines += [ + '**Orphan tests** (no feature file — behavior is tested but not documented):', + '', + ] + for tf in orphan_tests: + lines.append(f'- `{tf}`') + lines.append('') + + # ── recommendations (static) ───────────────────────────────────────────── + lines += ['---', '', _RECOMMENDATIONS, ''] + + # ── appendix ────────────────────────────────────────────────────────────── + lines += [ + '---', + '', + '## 7. Appendix: File Inventory', + '', + '### Feature Files', + '', + '| # | File | Scenarios |', + '|:---:|---|:---:|', + ] + for i, (feat, scen, *_) in enumerate(pairs, 1): + lines.append(f'| {i} | `{feat}` | {scen} |') + lines.append(f'| | **Total** | **{total_scenarios}** |') + lines.append('') + + lines += [ + '### Test Files', + '', + '| # | File | Tests |', + '|:---:|---|:---:|', + ] + all_tests = [(p[2], p[3]) for p in pairs if p[2]] + all_tests += list(orphan_tests.items()) + all_tests.sort() + for i, (tf, cnt) in enumerate(all_tests, 1): + lines.append(f'| {i} | `{tf}` | {cnt} |') + lines.append(f'| | **Total** | **{total_test_funcs}** |') + lines.append('') + + return '\n'.join(lines) + + +# --------------------------------------------------------------------------- +# entry point +# --------------------------------------------------------------------------- + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--tracefile', required=True) + ap.add_argument('--features-dir', required=True) + ap.add_argument('--tests-dir', required=True) + ap.add_argument('--output', required=True) + args = ap.parse_args() + + global features # used in build_mapping closure for orphan label + features = scan_features(args.features_dir) + tests = scan_tests(args.tests_dir) + cov_files, summary = parse_tracefile(args.tracefile) + pairs, orphan_feats, orphan_tests = build_mapping(features, tests) + today = datetime.date.today().strftime('%Y-%m-%d') + + md = generate(pairs, orphan_feats, orphan_tests, cov_files, summary, today) + + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, 'w') as fh: + fh.write(md) + + print(f'Written: {args.output}') + print(f' lines={summary["lines_pct"]} ' + f'branches={summary["branches_pct"]} ' + f'functions={summary["funcs_pct"]}') + + +if __name__ == '__main__': + main() diff --git a/test/functional-tests/tests/helper_functions.py b/test/functional-tests/tests/helper_functions.py index a6157e0f9..d4caa1a07 100644 --- a/test/functional-tests/tests/helper_functions.py +++ b/test/functional-tests/tests/helper_functions.py @@ -55,6 +55,8 @@ def remove_logfile(): def kill_rrd(signal: int=9): + if os.environ.get("RRD_COVERAGE_MODE") and signal == 9: + signal = 15 print(f"Received Signal to kill remotedebugger {signal} with pid {get_pid('remotedebugger')}") resp = subprocess.run(f"kill -{signal} {get_pid('remotedebugger')}", shell=True, capture_output=True) print(resp.stdout.decode('utf-8'))