From d246ab2bee684ba10430f0b707025494a099e48e Mon Sep 17 00:00:00 2001 From: dshett549 Date: Tue, 27 Jan 2026 08:06:33 +0530 Subject: [PATCH 1/9] RDK-60409: Component independent Tool Development for Noisy Log detection Signed-off-by: dshett549 --- .github/workflows/log-quality-rules.yml | 156 +++++++++++++++++++ noisylogdetector.py | 195 ++++++++++++++++++++++++ rules.yml | 51 +++++++ 3 files changed, 402 insertions(+) create mode 100644 .github/workflows/log-quality-rules.yml create mode 100644 noisylogdetector.py create mode 100644 rules.yml diff --git a/.github/workflows/log-quality-rules.yml b/.github/workflows/log-quality-rules.yml new file mode 100644 index 00000000..0a28cf5f --- /dev/null +++ b/.github/workflows/log-quality-rules.yml @@ -0,0 +1,156 @@ +name: log-quality-rules +on: + push: + branches: [ main, develop, release, support/*, hotfix/*, topic/* ] + pull_request: + branches: [ main, develop, release, support/*, hotfix/*, topic/* ] +jobs: + log-quality-check: + runs-on: ubuntu-22.04 + steps: + - name: Set up cache + id: cache + uses: actions/cache@v4 + with: + path: | + build + install + key: ${{ runner.os }}-${{ github.sha }} + + - name: Set up CMake + # 3.16.x To support flow control commands to be overridden. + # Installs in: find /opt -ipath *bin/cmake + uses: jwlawson/actions-setup-cmake@v1.12 + with: + cmake-version: '3.16.x' + - name: Install packages + run: > + sudo apt update && sudo apt install -y libcurl4-openssl-dev libgtest-dev lcov gcovr libmsgpack* libcjson-dev build-essential + - name: Checkout rbus + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + path: rbus + - name: Install dependencies + run: pip install pyyaml + - name: Configure Rbus + if: steps.cache.outputs.cache-hit != 'true' + run: > + cmake + -S "${{github.workspace}}/rbus" + -B build/rbus + -DCMAKE_INSTALL_PREFIX="${{github.workspace}}/install/usr" + -DBUILD_FOR_DESKTOP=ON + -DENABLE_UNIT_TESTING=ON + -DENABLE_CODE_COVERAGE=ON + -DCMAKE_BUILD_TYPE=Debug + - name: Build Rbus + if: steps.cache.outputs.cache-hit != 'true' + run: > + make -C build/rbus + - name: Install rbus + if: steps.cache.outputs.cache-hit != 'true' + run: > + make -C build/rbus install + - name: Start rtrouted + # nohup to keep it running. + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + nohup ./bin/rtrouted -f -l DEBUG > /tmp/rtrouted_log.txt & + - name: Run drop test with inactive provider + continue-on-error: true + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info + nohup ./bin/rbusSampleProvider > /tmp/log_sampleProvider.log 2>&1 & + sleep 1 + nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > /tmp/log_blockingProvider.log 2>&1 & + sleep 1 + ./bin/rbusDmlBlockingConsumer > /tmp/log_dmlBlockingConsumer.log 2>&1 + cat /tmp/log_sampleProvider.log || true + cat /tmp/log_blockingProvider.log || true + cat /tmp/log_dmlBlockingConsumer.log || true + - name: Run RbusTestTimeoutValues Unit test + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info + nohup ./bin/rbusTestTimeoutValuesProvider > /tmp/log_testTimeoutValuesProvider.txt & + ./bin/rbusTestTimeoutValuesConsumer > /tmp/log_testTimeoutValuesConsumer.txt + - name: Run Unit test + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info + nohup ./bin/rbusTestProvider >/tmp/log_TestAppProvider.txt & + ./bin/rbusTestConsumer -a > /tmp/log_TestConsumer.txt + - name: Run multiRbusOpenMethod Unit Test + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info + ./bin/multiRbusOpenMethodProvider > /tmp/log_MultiRbusOpenMethodProvider.txt & + ./bin/multiRbusOpenMethodConsumer > /tmp/log_MultiRbusOpenMethodConsumer.txt & + - name: Run multiRbusOpenSubscribe Unit test + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info + nohup ./bin/multiRbusOpenProvider >/tmp/log_multiRbusOpenProvider.txt & + ./bin/multiRbusOpenConsumer >/tmp/log_multiRbusOpenConsumer.txt + - name: Run multiRbusOpenGet Unit test + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info + nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusGetProvider.txt & + ./bin/multiRbusOpenRbusGetConsumer >/tmp/log_multiRbusOpenRbusGetConsumer.txt + - name: Run multiRbusOpenSet Unit test + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info + nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusSetProvider.txt & + ./bin/multiRbusOpenRbusSetConsumer >/tmp/log_multiRbusOpenRbusSetConsumer.txt + - name: Run Gtest Cases + run: | + cd build/rbus + export RT_LOG_LEVEL=info + nohup ./src/session_manager/rbus_session_mgr & + ./unittests/rbus_gtest.bin > /tmp/log_gtestlogs.txt + - name: Analyze logs + run: | + cd ${{github.workspace}}/rbus + set -- /tmp/log_*.txt + if [ ! -e "$1" ]; then + echo "No log files found matching /tmp/log_*.txt, skipping log analysis." + exit 0 + else + for f in "$@"; do + python3 noisylogdetector.py "$f" "/tmp/noisy_log_report_$(basename "$f" .txt).html" + done + fi + - name: Stop SessionManager + run: | + killall -15 rbus_session_mgr || true + - name: Stop rtrouted + run: | + cd install/usr + export PREFIX=$PWD + export LD_LIBRARY_PATH=$PREFIX/lib + ./bin/rtrouted_diag shutdown + - name: Upload the noisy log report to Pull request using actions + uses: actions/upload-artifact@v4 + with: + name: noisy-log-report + path: /tmp/noisy_log_report_*.html diff --git a/noisylogdetector.py b/noisylogdetector.py new file mode 100644 index 00000000..326827dc --- /dev/null +++ b/noisylogdetector.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 + +import re +import sys +import yaml +from html import escape +from pathlib import Path + +# ----------------------------- +def load_rules(path="rules.yml"): + try: + with open(path, "r") as f: + return yaml.safe_load(f) + except FileNotFoundError: + print(f"Rules file not found: {path}", file=sys.stderr) + sys.exit(1) + except PermissionError: + print(f"Permission denied while reading rules file: {path}", file=sys.stderr) + sys.exit(1) + except yaml.YAMLError as e: + print(f"Failed to parse YAML rules file '{path}': {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Unexpected error while loading rules from '{path}': {e}", file=sys.stderr) + sys.exit(1) + +# ----------------------------- +def starts_with_date_and_timestamp(line): + """ + Matches log lines starting with any of the following timestamp patterns: + - HH:MM:SS or HH:MM:SS.ssssss (e.g. 04:31:14 or 04:31:14.109764) + - YYYY-MM-DD HH:MM:SS or YYYY-MM-DD HH:MM:SS.sss (e.g. 2024-11-11 04:31:14 or 2024-11-11 04:31:14.109) + - Mon DD HH:MM:SS (e.g. Nov 11 04:31:14) + Lines not matching these patterns at the start will be ignored. + + NOTE: If your log lines are not being reported, check: + - The timestamp is at the very start of the line (no leading spaces/tabs). + - The timestamp matches one of the above formats. + - If there are leading spaces, adjust the regex to allow them. + """ + # This regex allows optional leading whitespace before the timestamp. + return bool(re.match( + r'^\s*(\d{2}:\d{2}:\d{2}(?:\.\d+)?|\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d+)?|' + r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})', + line + )) + +def detect_level(line): + for lvl in ("ERROR", "WARN", "INFO", "DEBUG", "TRACE"): + if re.search(rf"\b{lvl}\b", line): + return lvl + return "UNKNOWN" + +# ----------------------------- +def compile_patterns(patterns): + return [re.compile(p, re.IGNORECASE) for p in patterns] + +# ----------------------------- +def analyze(log_file, rules): + """ + Analyze a log file for noisy logging, sensitive data exposure, and + incorrect severity usage based on the provided rules. + Parameters + ---------- + log_file : str or pathlib.Path + Path to the log file to analyze. The file is opened in text mode + with errors ignored to allow processing partially invalid encodings. + rules : dict + Configuration dictionary containing analysis rules. Expected keys: + - "sensitive_patterns": list of regex patterns that match sensitive + or PII data that must not appear in logs. + - "failure_keywords": list of lowercase keywords that indicate a + failure or error condition in a log line. + - "noisy_log_levels": iterable of log levels (e.g. "INFO", "DEBUG") + that are considered noisy. + - "required_severity_on_failure": iterable of log levels (e.g. + "ERROR", "WARN") that must be used when a failure keyword is + present. + Returns + ------- + tuple + A 3-tuple `(noisy_logs, sensitive_logs, severity_violations)` where + each element is a list of dictionaries describing matching log lines. + - noisy_logs: entries for logs emitted at noisy log levels. + - sensitive_logs: entries where sensitive or PII data was detected, + with similar structure ("line", "log", "reason"). + - severity_violations: entries where a failure keyword was found but + the log level did not meet the required severity. + """ + noisy_logs = [] + sensitive_logs = [] + severity_violations = [] + + sensitive_res = compile_patterns(rules["sensitive_patterns"]) + failure_keywords = rules["failure_keywords"] + + with open(log_file, "r", errors="ignore") as f: + for ln, line in enumerate(f, 1): + line = line.rstrip() + + if not starts_with_date_and_timestamp(line): + continue + + level = detect_level(line) + + # Report all noisy log levels (DEBUG, TRACE, INFO) as noisy logs + if level in rules["noisy_log_levels"]: + noisy_logs.append({ + "line": ln, + "log": line, + "reason": f"Noisy log level: {level}" + }) + + # Sensitive logs + for r in sensitive_res: + if r.search(line): + sensitive_logs.append({ + "line": ln, + "log": line, + "reason": "Sensitive / PII data detected" + }) + break + + # Severity enforcement + if any(k in line.lower() for k in failure_keywords): + if level not in rules["required_severity_on_failure"]: + severity_violations.append({ + "line": ln, + "log": line, + "reason": "Failure logged without ERROR/WARN" + }) + + return noisy_logs, sensitive_logs, severity_violations + +# ----------------------------- +def generate_html(noisy, sensitive, severity, output): + with open(output, "w", encoding="utf-8") as f: + f.write(""" + + +Log Quality Report + + + +

Log Quality Report

+""") + + def write_section(title, rows): + f.write(f"

{title}

") + f.write("") + f.write("") + if not rows: + f.write('') + else: + for r in rows: + f.write( + f"" + f"" + f"" + ) + f.write("
LineReasonLog
No issues found in this section.
{r['line']}{escape(r['reason'])}{escape(r['log'])}
") + + write_section("Noisy Logs", noisy) + write_section("Sensitive / PII Logs", sensitive) + write_section("Severity Violations", severity) + + f.write("") + + print(f"Report generated: {output}") + +# ----------------------------- +if __name__ == "__main__": + if len(sys.argv) < 3: + print( + "Usage: python3 noisylogdetector.py \n" + "Note: requires rules.yml in the current working directory." + ) + sys.exit(1) + + log_file = sys.argv[1] + output = sys.argv[2] + + if not Path(log_file).exists(): + print(f"Log file not found: {log_file}") + sys.exit(1) + + rules = load_rules() + noisy, sensitive, severity = analyze(log_file, rules) + generate_html(noisy, sensitive, severity, output) + diff --git a/rules.yml b/rules.yml new file mode 100644 index 00000000..aa5ca586 --- /dev/null +++ b/rules.yml @@ -0,0 +1,51 @@ +# ------------------------------ +# Rules for log analyzer +# ------------------------------ + +# Patterns to detect sensitive information +sensitive_patterns: + # JWT tokens + - '(?i)\btoken\s*[:=]\s*[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\b' + + # API keys + - '(?i)\b(api[_-]?key|apikey)\s*[:=]\s*[A-Za-z0-9]{20,}\b' + + # Password assignments + - '(?i)\b(password|passwd|pwd)\s*[:=]\s*\S{6,}\b' + + # Authorization headers + - '(?i)\bauthorization\s*[:=]\s*(bearer|basic)\s+[A-Za-z0-9+/=._-]+\b' + + # AWS keys + - '\bAKIA[0-9A-Z]{16}\b' + + # Private key blocks + - '-----BEGIN .*PRIVATE KEY-----[\s\S]*?-----END .*PRIVATE KEY-----' + + # URLs + - '\bhttps?://[^\s]+' + + # IP addresses + - '\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b' + + # MAC addresses + - '\b([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})\b' + +# Log levels considered noisy +noisy_log_levels: + - DEBUG + - TRACE + - INFO + +# Keywords indicating failure +failure_keywords: + - 'failed' + - 'error' + - 'exception' + - 'invalid' + - 'timeout' + +# Required log levels for failure messages +required_severity_on_failure: + - ERROR + - WARN From a4def03417e7b251a00d628bf8f1a4b056edb57c Mon Sep 17 00:00:00 2001 From: dshett549 Date: Tue, 27 Jan 2026 09:32:12 +0530 Subject: [PATCH 2/9] Address Copilot review comments Signed-off-by: dshett549 --- noisylogdetector.py | 102 ++++++++++++++++++++++++++++++-------------- 1 file changed, 71 insertions(+), 31 deletions(-) diff --git a/noisylogdetector.py b/noisylogdetector.py index 326827dc..6450f962 100644 --- a/noisylogdetector.py +++ b/noisylogdetector.py @@ -34,7 +34,7 @@ def starts_with_date_and_timestamp(line): Lines not matching these patterns at the start will be ignored. NOTE: If your log lines are not being reported, check: - - The timestamp is at the very start of the line (no leading spaces/tabs). + - The timestamp is at the very start of the line. - The timestamp matches one of the above formats. - If there are leading spaces, adjust the regex to allow them. """ @@ -94,41 +94,81 @@ def analyze(log_file, rules): sensitive_res = compile_patterns(rules["sensitive_patterns"]) failure_keywords = rules["failure_keywords"] + # Read the full log file once so we can both: + # - Scan line-by-line (existing behavior), and + # - Apply sensitive patterns across multiple lines. with open(log_file, "r", errors="ignore") as f: - for ln, line in enumerate(f, 1): - line = line.rstrip() - - if not starts_with_date_and_timestamp(line): + full_text = f.read() + # Build a mapping from character offsets to line numbers. + line_starts = [] + current_line = 1 + line_starts.append((0, current_line)) + for idx, ch in enumerate(full_text): + if ch == "\n": + current_line += 1 + line_starts.append((idx + 1, current_line)) + def _offset_to_line(pos): + """ + Map a character offset in full_text to a 1-based line number using + the precomputed line_starts list. + """ + line_no = 1 + for start, ln in line_starts: + if start > pos: + break + line_no = ln + return line_no + # Split into lines for existing per-line analysis. + lines = full_text.splitlines() + # First, detect multi-line sensitive matches that would not be visible + # when scanning individual lines. + for r in sensitive_res: + for match in r.finditer(full_text): + matched_text = match.group(0) + # Only treat as multi-line if the match actually spans lines. + if "\n" not in matched_text and "\r" not in matched_text: continue - - level = detect_level(line) - - # Report all noisy log levels (DEBUG, TRACE, INFO) as noisy logs - if level in rules["noisy_log_levels"]: - noisy_logs.append({ + start_pos = match.start() + ln = _offset_to_line(start_pos) + if 1 <= ln <= len(lines): + line_text = lines[ln - 1] + else: + line_text = "" + sensitive_logs.append({ + "line": ln, + "log": line_text, + "reason": "Sensitive / PII data detected (multi-line match)" + }) + # Existing per-line analysis. + for ln, line in enumerate(lines, 1): + line = line.rstrip() + if not starts_with_date_and_timestamp(line): + continue + level = detect_level(line) + # Report all noisy log levels (DEBUG, TRACE, INFO) as noisy logs + if level in rules["noisy_log_levels"]: + noisy_logs.append({ + "line": ln, + "log": line, + "reason": f"Noisy log level: {level}" + }) + # Sensitive logs (single-line or line-contained matches) + for r in sensitive_res: + if r.search(line): + sensitive_logs.append({ "line": ln, "log": line, - "reason": f"Noisy log level: {level}" + "reason": "Sensitive / PII data detected" + }) + break + # Severity enforcement + if any(k in line.lower() for k in failure_keywords): + if level not in rules["required_severity_on_failure"]: + severity_violations.append({ + "line": ln, + "log": line, + "reason": "Failure logged without ERROR/WARN" }) - - # Sensitive logs - for r in sensitive_res: - if r.search(line): - sensitive_logs.append({ - "line": ln, - "log": line, - "reason": "Sensitive / PII data detected" - }) - break - - # Severity enforcement - if any(k in line.lower() for k in failure_keywords): - if level not in rules["required_severity_on_failure"]: - severity_violations.append({ - "line": ln, - "log": line, - "reason": "Failure logged without ERROR/WARN" - }) return noisy_logs, sensitive_logs, severity_violations From 398a890deb7012a5f0832b09308607d7339b2fcf Mon Sep 17 00:00:00 2001 From: dshett549 Date: Tue, 27 Jan 2026 10:02:07 +0530 Subject: [PATCH 3/9] Address copilot review comments Signed-off-by: dshett549 --- .github/workflows/log-quality-rules.yml | 12 +-- noisylogdetector.py | 98 +++++++------------------ 2 files changed, 33 insertions(+), 77 deletions(-) diff --git a/.github/workflows/log-quality-rules.yml b/.github/workflows/log-quality-rules.yml index 0a28cf5f..88c57295 100644 --- a/.github/workflows/log-quality-rules.yml +++ b/.github/workflows/log-quality-rules.yml @@ -66,14 +66,14 @@ jobs: export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib export RT_LOG_LEVEL=info - nohup ./bin/rbusSampleProvider > /tmp/log_sampleProvider.log 2>&1 & + nohup ./bin/rbusSampleProvider > /tmp/log_sampleProvider.txt 2>&1 & sleep 1 - nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > /tmp/log_blockingProvider.log 2>&1 & + nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > /tmp/log_blockingProvider.txt 2>&1 & sleep 1 - ./bin/rbusDmlBlockingConsumer > /tmp/log_dmlBlockingConsumer.log 2>&1 - cat /tmp/log_sampleProvider.log || true - cat /tmp/log_blockingProvider.log || true - cat /tmp/log_dmlBlockingConsumer.log || true + ./bin/rbusDmlBlockingConsumer > /tmp/log_dmlBlockingConsumer.txt 2>&1 + cat /tmp/log_sampleProvider.txt || true + cat /tmp/log_blockingProvider.txt || true + cat /tmp/log_dmlBlockingConsumer.txt || true - name: Run RbusTestTimeoutValues Unit test run: | cd install/usr diff --git a/noisylogdetector.py b/noisylogdetector.py index 6450f962..9b8e3090 100644 --- a/noisylogdetector.py +++ b/noisylogdetector.py @@ -27,7 +27,7 @@ def load_rules(path="rules.yml"): # ----------------------------- def starts_with_date_and_timestamp(line): """ - Matches log lines starting with any of the following timestamp patterns: + Matches log lines starting with any of the following timestamp patterns including leading whitespaces: - HH:MM:SS or HH:MM:SS.ssssss (e.g. 04:31:14 or 04:31:14.109764) - YYYY-MM-DD HH:MM:SS or YYYY-MM-DD HH:MM:SS.sss (e.g. 2024-11-11 04:31:14 or 2024-11-11 04:31:14.109) - Mon DD HH:MM:SS (e.g. Nov 11 04:31:14) @@ -94,81 +94,37 @@ def analyze(log_file, rules): sensitive_res = compile_patterns(rules["sensitive_patterns"]) failure_keywords = rules["failure_keywords"] - # Read the full log file once so we can both: - # - Scan line-by-line (existing behavior), and - # - Apply sensitive patterns across multiple lines. + # - Scan line-by-line with open(log_file, "r", errors="ignore") as f: - full_text = f.read() - # Build a mapping from character offsets to line numbers. - line_starts = [] - current_line = 1 - line_starts.append((0, current_line)) - for idx, ch in enumerate(full_text): - if ch == "\n": - current_line += 1 - line_starts.append((idx + 1, current_line)) - def _offset_to_line(pos): - """ - Map a character offset in full_text to a 1-based line number using - the precomputed line_starts list. - """ - line_no = 1 - for start, ln in line_starts: - if start > pos: - break - line_no = ln - return line_no - # Split into lines for existing per-line analysis. - lines = full_text.splitlines() - # First, detect multi-line sensitive matches that would not be visible - # when scanning individual lines. - for r in sensitive_res: - for match in r.finditer(full_text): - matched_text = match.group(0) - # Only treat as multi-line if the match actually spans lines. - if "\n" not in matched_text and "\r" not in matched_text: + for ln, line in enumerate(f, 1): + line = line.rstrip() + if not starts_with_date_and_timestamp(line): continue - start_pos = match.start() - ln = _offset_to_line(start_pos) - if 1 <= ln <= len(lines): - line_text = lines[ln - 1] - else: - line_text = "" - sensitive_logs.append({ - "line": ln, - "log": line_text, - "reason": "Sensitive / PII data detected (multi-line match)" - }) - # Existing per-line analysis. - for ln, line in enumerate(lines, 1): - line = line.rstrip() - if not starts_with_date_and_timestamp(line): - continue - level = detect_level(line) - # Report all noisy log levels (DEBUG, TRACE, INFO) as noisy logs - if level in rules["noisy_log_levels"]: - noisy_logs.append({ - "line": ln, - "log": line, - "reason": f"Noisy log level: {level}" - }) - # Sensitive logs (single-line or line-contained matches) - for r in sensitive_res: - if r.search(line): - sensitive_logs.append({ - "line": ln, - "log": line, - "reason": "Sensitive / PII data detected" - }) - break - # Severity enforcement - if any(k in line.lower() for k in failure_keywords): - if level not in rules["required_severity_on_failure"]: - severity_violations.append({ + level = detect_level(line) + # Report all noisy log levels (DEBUG, TRACE, INFO) as noisy logs + if level in rules["noisy_log_levels"]: + noisy_logs.append({ "line": ln, "log": line, - "reason": "Failure logged without ERROR/WARN" + "reason": f"Noisy log level: {level}" }) + # Sensitive logs + for r in sensitive_res: + if r.search(line): + sensitive_logs.append({ + "line": ln, + "log": line, + "reason": "Sensitive / PII data detected" + }) + break + # Severity enforcement + if any(k in line.lower() for k in failure_keywords): + if level not in rules["required_severity_on_failure"]: + severity_violations.append({ + "line": ln, + "log": line, + "reason": "Failure logged without ERROR/WARN" + }) return noisy_logs, sensitive_logs, severity_violations From 564aa94096ca2423501c5fbf48110aed91b6be7b Mon Sep 17 00:00:00 2001 From: dshett549 Date: Tue, 27 Jan 2026 10:27:50 +0530 Subject: [PATCH 4/9] Update log-quality-rules.yml Signed-off-by: dshett549 --- .github/workflows/log-quality-rules.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/log-quality-rules.yml b/.github/workflows/log-quality-rules.yml index 88c57295..219fcabb 100644 --- a/.github/workflows/log-quality-rules.yml +++ b/.github/workflows/log-quality-rules.yml @@ -89,6 +89,7 @@ jobs: export LD_LIBRARY_PATH=$PREFIX/lib export RT_LOG_LEVEL=info nohup ./bin/rbusTestProvider >/tmp/log_TestAppProvider.txt & + sleep 2 ./bin/rbusTestConsumer -a > /tmp/log_TestConsumer.txt - name: Run multiRbusOpenMethod Unit Test run: | From baa726bb4fbbe7211e91c1d066d10838282444c7 Mon Sep 17 00:00:00 2001 From: dshett549 Date: Tue, 27 Jan 2026 11:27:10 +0530 Subject: [PATCH 5/9] Address Copilot review comments Signed-off-by: dshett549 --- .github/workflows/log-quality-rules.yml | 157 ------------------------ .github/workflows/unit_tests.yml | 68 ++++++---- noisylogdetector.py | 28 ++++- rules.yml | 2 +- 4 files changed, 73 insertions(+), 182 deletions(-) delete mode 100644 .github/workflows/log-quality-rules.yml diff --git a/.github/workflows/log-quality-rules.yml b/.github/workflows/log-quality-rules.yml deleted file mode 100644 index 219fcabb..00000000 --- a/.github/workflows/log-quality-rules.yml +++ /dev/null @@ -1,157 +0,0 @@ -name: log-quality-rules -on: - push: - branches: [ main, develop, release, support/*, hotfix/*, topic/* ] - pull_request: - branches: [ main, develop, release, support/*, hotfix/*, topic/* ] -jobs: - log-quality-check: - runs-on: ubuntu-22.04 - steps: - - name: Set up cache - id: cache - uses: actions/cache@v4 - with: - path: | - build - install - key: ${{ runner.os }}-${{ github.sha }} - - - name: Set up CMake - # 3.16.x To support flow control commands to be overridden. - # Installs in: find /opt -ipath *bin/cmake - uses: jwlawson/actions-setup-cmake@v1.12 - with: - cmake-version: '3.16.x' - - name: Install packages - run: > - sudo apt update && sudo apt install -y libcurl4-openssl-dev libgtest-dev lcov gcovr libmsgpack* libcjson-dev build-essential - - name: Checkout rbus - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - path: rbus - - name: Install dependencies - run: pip install pyyaml - - name: Configure Rbus - if: steps.cache.outputs.cache-hit != 'true' - run: > - cmake - -S "${{github.workspace}}/rbus" - -B build/rbus - -DCMAKE_INSTALL_PREFIX="${{github.workspace}}/install/usr" - -DBUILD_FOR_DESKTOP=ON - -DENABLE_UNIT_TESTING=ON - -DENABLE_CODE_COVERAGE=ON - -DCMAKE_BUILD_TYPE=Debug - - name: Build Rbus - if: steps.cache.outputs.cache-hit != 'true' - run: > - make -C build/rbus - - name: Install rbus - if: steps.cache.outputs.cache-hit != 'true' - run: > - make -C build/rbus install - - name: Start rtrouted - # nohup to keep it running. - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rtrouted -f -l DEBUG > /tmp/rtrouted_log.txt & - - name: Run drop test with inactive provider - continue-on-error: true - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - export RT_LOG_LEVEL=info - nohup ./bin/rbusSampleProvider > /tmp/log_sampleProvider.txt 2>&1 & - sleep 1 - nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > /tmp/log_blockingProvider.txt 2>&1 & - sleep 1 - ./bin/rbusDmlBlockingConsumer > /tmp/log_dmlBlockingConsumer.txt 2>&1 - cat /tmp/log_sampleProvider.txt || true - cat /tmp/log_blockingProvider.txt || true - cat /tmp/log_dmlBlockingConsumer.txt || true - - name: Run RbusTestTimeoutValues Unit test - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - export RT_LOG_LEVEL=info - nohup ./bin/rbusTestTimeoutValuesProvider > /tmp/log_testTimeoutValuesProvider.txt & - ./bin/rbusTestTimeoutValuesConsumer > /tmp/log_testTimeoutValuesConsumer.txt - - name: Run Unit test - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - export RT_LOG_LEVEL=info - nohup ./bin/rbusTestProvider >/tmp/log_TestAppProvider.txt & - sleep 2 - ./bin/rbusTestConsumer -a > /tmp/log_TestConsumer.txt - - name: Run multiRbusOpenMethod Unit Test - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - export RT_LOG_LEVEL=info - ./bin/multiRbusOpenMethodProvider > /tmp/log_MultiRbusOpenMethodProvider.txt & - ./bin/multiRbusOpenMethodConsumer > /tmp/log_MultiRbusOpenMethodConsumer.txt & - - name: Run multiRbusOpenSubscribe Unit test - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - export RT_LOG_LEVEL=info - nohup ./bin/multiRbusOpenProvider >/tmp/log_multiRbusOpenProvider.txt & - ./bin/multiRbusOpenConsumer >/tmp/log_multiRbusOpenConsumer.txt - - name: Run multiRbusOpenGet Unit test - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - export RT_LOG_LEVEL=info - nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusGetProvider.txt & - ./bin/multiRbusOpenRbusGetConsumer >/tmp/log_multiRbusOpenRbusGetConsumer.txt - - name: Run multiRbusOpenSet Unit test - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - export RT_LOG_LEVEL=info - nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusSetProvider.txt & - ./bin/multiRbusOpenRbusSetConsumer >/tmp/log_multiRbusOpenRbusSetConsumer.txt - - name: Run Gtest Cases - run: | - cd build/rbus - export RT_LOG_LEVEL=info - nohup ./src/session_manager/rbus_session_mgr & - ./unittests/rbus_gtest.bin > /tmp/log_gtestlogs.txt - - name: Analyze logs - run: | - cd ${{github.workspace}}/rbus - set -- /tmp/log_*.txt - if [ ! -e "$1" ]; then - echo "No log files found matching /tmp/log_*.txt, skipping log analysis." - exit 0 - else - for f in "$@"; do - python3 noisylogdetector.py "$f" "/tmp/noisy_log_report_$(basename "$f" .txt).html" - done - fi - - name: Stop SessionManager - run: | - killall -15 rbus_session_mgr || true - - name: Stop rtrouted - run: | - cd install/usr - export PREFIX=$PWD - export LD_LIBRARY_PATH=$PREFIX/lib - ./bin/rtrouted_diag shutdown - - name: Upload the noisy log report to Pull request using actions - uses: actions/upload-artifact@v4 - with: - name: noisy-log-report - path: /tmp/noisy_log_report_*.html diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5b74f425..0b7991e1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -31,6 +31,8 @@ jobs: uses: actions/checkout@v3 with: path: rbus + - name: Install dependencies + run: pip install pyyaml - name: Configure Rbus if: steps.cache.outputs.cache-hit != 'true' run: > @@ -63,70 +65,79 @@ jobs: cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rbusSampleProvider > sampleProvider.log 2>&1 & + export RT_LOG_LEVEL=info + nohup ./bin/rbusSampleProvider > /tmp/log_sampleProvider.txt 2>&1 & sleep 1 - nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > blockingProvider.log 2>&1 & + nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > /tmp/log_blockingProvider.txt 2>&1 & sleep 1 - ./bin/rbusDmlBlockingConsumer > dmlBlockingConsumer.log 2>&1 - cat sampleProvider.log || true - cat blockingProvider.log || true - cat dmlBlockingConsumer.log || true + ./bin/rbusDmlBlockingConsumer > /tmp/log_dmlBlockingConsumer.txt 2>&1 + cat /tmp/log_sampleProvider.txt || true + cat /tmp/log_blockingProvider.txt || true + cat /tmp/log_dmlBlockingConsumer.txt || true - name: Run RbusTestTimeoutValues Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rbusTestTimeoutValuesProvider & - ./bin/rbusTestTimeoutValuesConsumer + export RT_LOG_LEVEL=info + nohup ./bin/rbusTestTimeoutValuesProvider > /tmp/log_testTimeoutValuesProvider.txt & + ./bin/rbusTestTimeoutValuesConsumer > /tmp/log_testTimeoutValuesConsumer.txt - name: Run Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rbusTestProvider >/tmp/plog.txt & - ./bin/rbusTestConsumer -a + export RT_LOG_LEVEL=info + nohup ./bin/rbusTestProvider >/tmp/log_TestAppProvider.txt & + sleep 2 + ./bin/rbusTestConsumer -a > /tmp/log_TestConsumer.txt - name: Run multiRbusOpenMethod Unit Test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - ./bin/multiRbusOpenMethodProvider & - ./bin/multiRbusOpenMethodConsumer & + export RT_LOG_LEVEL=info + ./bin/multiRbusOpenMethodProvider > /tmp/log_MultiRbusOpenMethodProvider.txt & + ./bin/multiRbusOpenMethodConsumer > /tmp/log_MultiRbusOpenMethodConsumer.txt & - name: Run multiRbusOpenSubscribe Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info nohup ./bin/multiRbusOpenProvider >/tmp/log_multiRbusOpenProvider.txt & - ./bin/multiRbusOpenConsumer + ./bin/multiRbusOpenConsumer >/tmp/log_multiRbusOpenConsumer.txt - name: Run multiRbusOpenGet Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusGetProvider.txt & - ./bin/multiRbusOpenRbusGetConsumer + ./bin/multiRbusOpenRbusGetConsumer >/tmp/log_multiRbusOpenRbusGetConsumer.txt - name: Run multiRbusOpenSet Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusSetProvider.txt & - ./bin/multiRbusOpenRbusSetConsumer + ./bin/multiRbusOpenRbusSetConsumer >/tmp/log_multiRbusOpenRbusSetConsumer.txt - name: Run Gtest Cases run: | cd build/rbus + export RT_LOG_LEVEL=info nohup ./src/session_manager/rbus_session_mgr & - ./unittests/rbus_gtest.bin + ./unittests/rbus_gtest.bin > /tmp/log_gtestlogs.txt - name: Stop SessionManager run: | - killall -15 rbus_session_mgr + killall -15 rbus_session_mgr || true - name: Stop rtrouted run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rtrouted_diag shutdown + ./bin/rtrouted_diag shutdown - name: Run CodeCoverage run: | rm -rf /tmp/rtrouted* @@ -135,10 +146,25 @@ jobs: - name: Generate the html report run: | genhtml filtered-coverage.info --output-directory /tmp/coverage_report - - - name: Upload the coverage report to Pull request using actions + - name: Upload the coverage report to Pull request using actions uses: actions/upload-artifact@v4 with: name: coverage-report path: /tmp/coverage_report - + - name: Analyze logs + run: | + cd ${{github.workspace}}/rbus + set -- /tmp/log_*.txt + if [ ! -e "$1" ]; then + echo "No log files found matching /tmp/log_*.txt, skipping log analysis." + exit 0 + else + for f in "$@"; do + python3 noisylogdetector.py "$f" "/tmp/noisy_log_report_$(basename "$f" .txt).html" + done + fi + - name: Upload the noisy log report to Pull request using actions + uses: actions/upload-artifact@v4 + with: + name: noisy-log-report + path: /tmp/noisy_log_report_*.html diff --git a/noisylogdetector.py b/noisylogdetector.py index 9b8e3090..b9753f64 100644 --- a/noisylogdetector.py +++ b/noisylogdetector.py @@ -10,7 +10,7 @@ def load_rules(path="rules.yml"): try: with open(path, "r") as f: - return yaml.safe_load(f) + rules= yaml.safe_load(f) except FileNotFoundError: print(f"Rules file not found: {path}", file=sys.stderr) sys.exit(1) @@ -23,6 +23,19 @@ def load_rules(path="rules.yml"): except Exception as e: print(f"Unexpected error while loading rules from '{path}': {e}", file=sys.stderr) sys.exit(1) + # --- Validate required keys --- + required_keys = [ + "sensitive_patterns", + "failure_keywords", + "noisy_log_levels", + "required_severity_on_failure" + ] + missing = [k for k in required_keys if k not in rules or rules[k] is None] + if missing: + print(f"Error: rules.yml is missing required keys: {', '.join(missing)}", file=sys.stderr) + sys.exit(1) + + return rules # ----------------------------- def starts_with_date_and_timestamp(line): @@ -94,6 +107,12 @@ def analyze(log_file, rules): sensitive_res = compile_patterns(rules["sensitive_patterns"]) failure_keywords = rules["failure_keywords"] + def redact_sensitive(line): + # Replace all sensitive matches with [REDACTED] + for r in sensitive_res: + line = r.sub("[REDACTED]", line) + return line + # - Scan line-by-line with open(log_file, "r", errors="ignore") as f: for ln, line in enumerate(f, 1): @@ -113,7 +132,7 @@ def analyze(log_file, rules): if r.search(line): sensitive_logs.append({ "line": ln, - "log": line, + "log": redact_sensitive(line), "reason": "Sensitive / PII data detected" }) break @@ -123,7 +142,10 @@ def analyze(log_file, rules): severity_violations.append({ "line": ln, "log": line, - "reason": "Failure logged without ERROR/WARN" + "reason": ( + "Failure logged without required severity: " + + ", ".join(rules["required_severity_on_failure"]) + ) }) return noisy_logs, sensitive_logs, severity_violations diff --git a/rules.yml b/rules.yml index aa5ca586..654b1d21 100644 --- a/rules.yml +++ b/rules.yml @@ -20,7 +20,7 @@ sensitive_patterns: - '\bAKIA[0-9A-Z]{16}\b' # Private key blocks - - '-----BEGIN .*PRIVATE KEY-----[\s\S]*?-----END .*PRIVATE KEY-----' + - '-----BEGIN .*PRIVATE KEY-----' # URLs - '\bhttps?://[^\s]+' From 130ea4db1dd80ef1c486f1084feb9c8763a8d37a Mon Sep 17 00:00:00 2001 From: Deepthi C Shetty <115452109+dshett549@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:38:40 +0530 Subject: [PATCH 6/9] Update unit_tests.yml --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 0b7991e1..7e1d6d83 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -146,7 +146,7 @@ jobs: - name: Generate the html report run: | genhtml filtered-coverage.info --output-directory /tmp/coverage_report - - name: Upload the coverage report to Pull request using actions + - name: Upload the coverage report to Pull request using actions uses: actions/upload-artifact@v4 with: name: coverage-report From faad99d4e0f9ed6941a6be924c15dea5e097ecff Mon Sep 17 00:00:00 2001 From: dshett549 Date: Tue, 27 Jan 2026 12:11:58 +0530 Subject: [PATCH 7/9] Update noisyLogDetector.py Signed-off-by: dshett549 --- noisylogdetector.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/noisylogdetector.py b/noisylogdetector.py index b9753f64..0d2be613 100644 --- a/noisylogdetector.py +++ b/noisylogdetector.py @@ -23,6 +23,10 @@ def load_rules(path="rules.yml"): except Exception as e: print(f"Unexpected error while loading rules from '{path}': {e}", file=sys.stderr) sys.exit(1) + # --- Validate rules is a dict --- + if not isinstance(rules, dict): + print(f"Error: rules.yml is empty or not a valid YAML mapping.", file=sys.stderr) + sys.exit(1) # --- Validate required keys --- required_keys = [ "sensitive_patterns", @@ -59,14 +63,14 @@ def starts_with_date_and_timestamp(line): )) def detect_level(line): - for lvl in ("ERROR", "WARN", "INFO", "DEBUG", "TRACE"): + for lvl in ("FATAL","ERROR", "WARN", "INFO", "DEBUG", "TRACE"): if re.search(rf"\b{lvl}\b", line): return lvl return "UNKNOWN" # ----------------------------- def compile_patterns(patterns): - return [re.compile(p, re.IGNORECASE) for p in patterns] + return [re.compile(p) for p in patterns] # ----------------------------- def analyze(log_file, rules): @@ -124,7 +128,7 @@ def redact_sensitive(line): if level in rules["noisy_log_levels"]: noisy_logs.append({ "line": ln, - "log": line, + "log": redact_sensitive(line), "reason": f"Noisy log level: {level}" }) # Sensitive logs @@ -141,7 +145,7 @@ def redact_sensitive(line): if level not in rules["required_severity_on_failure"]: severity_violations.append({ "line": ln, - "log": line, + "log": redact_sensitive(line), "reason": ( "Failure logged without required severity: " + ", ".join(rules["required_severity_on_failure"]) From 3cf89f30446a59ccc2ae37dbdb14833d446bb576 Mon Sep 17 00:00:00 2001 From: dshett549 Date: Tue, 14 Apr 2026 15:59:06 +0530 Subject: [PATCH 8/9] testing Signed-off-by: dshett549 --- src/rbus/rbus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rbus/rbus.c b/src/rbus/rbus.c index dfed4e50..261e77ac 100644 --- a/src/rbus/rbus.c +++ b/src/rbus/rbus.c @@ -2794,7 +2794,7 @@ static void _create_direct_connection_callback_handler (rbusHandle_t handle, rbu // PID is something unique but the same client can have direct connection to two different providers and that could lead to failure. // ex: if the consumer pid is 12345; - // Provider1 may open a private session on 127.0.0.1:12345 + // Provider1 may open a private session on 127.0.0.1:12345 // Provider2 should not (could not) open private session on same address as 127.0.0.1:12345. // Provider 2 have to come up with some other logic From 7b9743a655b180e63a50dde203e64cf064ed64ef Mon Sep 17 00:00:00 2001 From: dshett549 Date: Wed, 15 Apr 2026 08:19:26 +0530 Subject: [PATCH 9/9] test Signed-off-by: dshett549 --- src/rbus/rbus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rbus/rbus.c b/src/rbus/rbus.c index 261e77ac..ad72b045 100644 --- a/src/rbus/rbus.c +++ b/src/rbus/rbus.c @@ -2794,7 +2794,7 @@ static void _create_direct_connection_callback_handler (rbusHandle_t handle, rbu // PID is something unique but the same client can have direct connection to two different providers and that could lead to failure. // ex: if the consumer pid is 12345; - // Provider1 may open a private session on 127.0.0.1:12345 + // Provider1 may open a i private session on 127.0.0.1:12345 // Provider2 should not (could not) open private session on same address as 127.0.0.1:12345. // Provider 2 have to come up with some other logic