diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5b74f425..7e1d6d83 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -31,6 +31,8 @@ jobs: uses: actions/checkout@v3 with: path: rbus + - name: Install dependencies + run: pip install pyyaml - name: Configure Rbus if: steps.cache.outputs.cache-hit != 'true' run: > @@ -63,70 +65,79 @@ jobs: cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rbusSampleProvider > sampleProvider.log 2>&1 & + export RT_LOG_LEVEL=info + nohup ./bin/rbusSampleProvider > /tmp/log_sampleProvider.txt 2>&1 & sleep 1 - nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > blockingProvider.log 2>&1 & + nohup stdbuf -oL -eL ./bin/rbusBlockingProvider > /tmp/log_blockingProvider.txt 2>&1 & sleep 1 - ./bin/rbusDmlBlockingConsumer > dmlBlockingConsumer.log 2>&1 - cat sampleProvider.log || true - cat blockingProvider.log || true - cat dmlBlockingConsumer.log || true + ./bin/rbusDmlBlockingConsumer > /tmp/log_dmlBlockingConsumer.txt 2>&1 + cat /tmp/log_sampleProvider.txt || true + cat /tmp/log_blockingProvider.txt || true + cat /tmp/log_dmlBlockingConsumer.txt || true - name: Run RbusTestTimeoutValues Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rbusTestTimeoutValuesProvider & - ./bin/rbusTestTimeoutValuesConsumer + export RT_LOG_LEVEL=info + nohup ./bin/rbusTestTimeoutValuesProvider > /tmp/log_testTimeoutValuesProvider.txt & + ./bin/rbusTestTimeoutValuesConsumer > /tmp/log_testTimeoutValuesConsumer.txt - name: Run Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rbusTestProvider >/tmp/plog.txt & - ./bin/rbusTestConsumer -a + export RT_LOG_LEVEL=info + nohup ./bin/rbusTestProvider >/tmp/log_TestAppProvider.txt & + sleep 2 + ./bin/rbusTestConsumer -a > /tmp/log_TestConsumer.txt - name: Run multiRbusOpenMethod Unit Test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - ./bin/multiRbusOpenMethodProvider & - ./bin/multiRbusOpenMethodConsumer & + export RT_LOG_LEVEL=info + ./bin/multiRbusOpenMethodProvider > /tmp/log_MultiRbusOpenMethodProvider.txt & + ./bin/multiRbusOpenMethodConsumer > /tmp/log_MultiRbusOpenMethodConsumer.txt & - name: Run multiRbusOpenSubscribe Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info nohup ./bin/multiRbusOpenProvider >/tmp/log_multiRbusOpenProvider.txt & - ./bin/multiRbusOpenConsumer + ./bin/multiRbusOpenConsumer >/tmp/log_multiRbusOpenConsumer.txt - name: Run multiRbusOpenGet Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusGetProvider.txt & - ./bin/multiRbusOpenRbusGetConsumer + ./bin/multiRbusOpenRbusGetConsumer >/tmp/log_multiRbusOpenRbusGetConsumer.txt - name: Run multiRbusOpenSet Unit test run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib + export RT_LOG_LEVEL=info nohup ./bin/multiRbusOpenRbusGetProvider >/tmp/log_multiRbusOpenRbusSetProvider.txt & - ./bin/multiRbusOpenRbusSetConsumer + ./bin/multiRbusOpenRbusSetConsumer >/tmp/log_multiRbusOpenRbusSetConsumer.txt - name: Run Gtest Cases run: | cd build/rbus + export RT_LOG_LEVEL=info nohup ./src/session_manager/rbus_session_mgr & - ./unittests/rbus_gtest.bin + ./unittests/rbus_gtest.bin > /tmp/log_gtestlogs.txt - name: Stop SessionManager run: | - killall -15 rbus_session_mgr + killall -15 rbus_session_mgr || true - name: Stop rtrouted run: | cd install/usr export PREFIX=$PWD export LD_LIBRARY_PATH=$PREFIX/lib - nohup ./bin/rtrouted_diag shutdown + ./bin/rtrouted_diag shutdown - name: Run CodeCoverage run: | rm -rf /tmp/rtrouted* @@ -135,10 +146,25 @@ jobs: - name: Generate the html report run: | genhtml filtered-coverage.info --output-directory /tmp/coverage_report - - name: Upload the coverage report to Pull request using actions uses: actions/upload-artifact@v4 with: name: coverage-report path: /tmp/coverage_report - + - name: Analyze logs + run: | + cd ${{github.workspace}}/rbus + set -- /tmp/log_*.txt + if [ ! -e "$1" ]; then + echo "No log files found matching /tmp/log_*.txt, skipping log analysis." + exit 0 + else + for f in "$@"; do + python3 noisylogdetector.py "$f" "/tmp/noisy_log_report_$(basename "$f" .txt).html" + done + fi + - name: Upload the noisy log report to Pull request using actions + uses: actions/upload-artifact@v4 + with: + name: noisy-log-report + path: /tmp/noisy_log_report_*.html diff --git a/noisylogdetector.py b/noisylogdetector.py new file mode 100644 index 00000000..0d2be613 --- /dev/null +++ b/noisylogdetector.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 + +import re +import sys +import yaml +from html import escape +from pathlib import Path + +# ----------------------------- +def load_rules(path="rules.yml"): + try: + with open(path, "r") as f: + rules= yaml.safe_load(f) + except FileNotFoundError: + print(f"Rules file not found: {path}", file=sys.stderr) + sys.exit(1) + except PermissionError: + print(f"Permission denied while reading rules file: {path}", file=sys.stderr) + sys.exit(1) + except yaml.YAMLError as e: + print(f"Failed to parse YAML rules file '{path}': {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Unexpected error while loading rules from '{path}': {e}", file=sys.stderr) + sys.exit(1) + # --- Validate rules is a dict --- + if not isinstance(rules, dict): + print(f"Error: rules.yml is empty or not a valid YAML mapping.", file=sys.stderr) + sys.exit(1) + # --- Validate required keys --- + required_keys = [ + "sensitive_patterns", + "failure_keywords", + "noisy_log_levels", + "required_severity_on_failure" + ] + missing = [k for k in required_keys if k not in rules or rules[k] is None] + if missing: + print(f"Error: rules.yml is missing required keys: {', '.join(missing)}", file=sys.stderr) + sys.exit(1) + + return rules + +# ----------------------------- +def starts_with_date_and_timestamp(line): + """ + Matches log lines starting with any of the following timestamp patterns including leading whitespaces: + - HH:MM:SS or HH:MM:SS.ssssss (e.g. 04:31:14 or 04:31:14.109764) + - YYYY-MM-DD HH:MM:SS or YYYY-MM-DD HH:MM:SS.sss (e.g. 2024-11-11 04:31:14 or 2024-11-11 04:31:14.109) + - Mon DD HH:MM:SS (e.g. Nov 11 04:31:14) + Lines not matching these patterns at the start will be ignored. + + NOTE: If your log lines are not being reported, check: + - The timestamp is at the very start of the line. + - The timestamp matches one of the above formats. + - If there are leading spaces, adjust the regex to allow them. + """ + # This regex allows optional leading whitespace before the timestamp. + return bool(re.match( + r'^\s*(\d{2}:\d{2}:\d{2}(?:\.\d+)?|\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d+)?|' + r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})', + line + )) + +def detect_level(line): + for lvl in ("FATAL","ERROR", "WARN", "INFO", "DEBUG", "TRACE"): + if re.search(rf"\b{lvl}\b", line): + return lvl + return "UNKNOWN" + +# ----------------------------- +def compile_patterns(patterns): + return [re.compile(p) for p in patterns] + +# ----------------------------- +def analyze(log_file, rules): + """ + Analyze a log file for noisy logging, sensitive data exposure, and + incorrect severity usage based on the provided rules. + Parameters + ---------- + log_file : str or pathlib.Path + Path to the log file to analyze. The file is opened in text mode + with errors ignored to allow processing partially invalid encodings. + rules : dict + Configuration dictionary containing analysis rules. Expected keys: + - "sensitive_patterns": list of regex patterns that match sensitive + or PII data that must not appear in logs. + - "failure_keywords": list of lowercase keywords that indicate a + failure or error condition in a log line. + - "noisy_log_levels": iterable of log levels (e.g. "INFO", "DEBUG") + that are considered noisy. + - "required_severity_on_failure": iterable of log levels (e.g. + "ERROR", "WARN") that must be used when a failure keyword is + present. + Returns + ------- + tuple + A 3-tuple `(noisy_logs, sensitive_logs, severity_violations)` where + each element is a list of dictionaries describing matching log lines. + - noisy_logs: entries for logs emitted at noisy log levels. + - sensitive_logs: entries where sensitive or PII data was detected, + with similar structure ("line", "log", "reason"). + - severity_violations: entries where a failure keyword was found but + the log level did not meet the required severity. + """ + noisy_logs = [] + sensitive_logs = [] + severity_violations = [] + + sensitive_res = compile_patterns(rules["sensitive_patterns"]) + failure_keywords = rules["failure_keywords"] + + def redact_sensitive(line): + # Replace all sensitive matches with [REDACTED] + for r in sensitive_res: + line = r.sub("[REDACTED]", line) + return line + + # - Scan line-by-line + with open(log_file, "r", errors="ignore") as f: + for ln, line in enumerate(f, 1): + line = line.rstrip() + if not starts_with_date_and_timestamp(line): + continue + level = detect_level(line) + # Report all noisy log levels (DEBUG, TRACE, INFO) as noisy logs + if level in rules["noisy_log_levels"]: + noisy_logs.append({ + "line": ln, + "log": redact_sensitive(line), + "reason": f"Noisy log level: {level}" + }) + # Sensitive logs + for r in sensitive_res: + if r.search(line): + sensitive_logs.append({ + "line": ln, + "log": redact_sensitive(line), + "reason": "Sensitive / PII data detected" + }) + break + # Severity enforcement + if any(k in line.lower() for k in failure_keywords): + if level not in rules["required_severity_on_failure"]: + severity_violations.append({ + "line": ln, + "log": redact_sensitive(line), + "reason": ( + "Failure logged without required severity: " + + ", ".join(rules["required_severity_on_failure"]) + ) + }) + + return noisy_logs, sensitive_logs, severity_violations + +# ----------------------------- +def generate_html(noisy, sensitive, severity, output): + with open(output, "w", encoding="utf-8") as f: + f.write(""" + + +Log Quality Report + + + +

Log Quality Report

+""") + + def write_section(title, rows): + f.write(f"

{title}

") + f.write("") + f.write("") + if not rows: + f.write('') + else: + for r in rows: + f.write( + f"" + f"" + f"" + ) + f.write("
LineReasonLog
No issues found in this section.
{r['line']}{escape(r['reason'])}{escape(r['log'])}
") + + write_section("Noisy Logs", noisy) + write_section("Sensitive / PII Logs", sensitive) + write_section("Severity Violations", severity) + + f.write("") + + print(f"Report generated: {output}") + +# ----------------------------- +if __name__ == "__main__": + if len(sys.argv) < 3: + print( + "Usage: python3 noisylogdetector.py \n" + "Note: requires rules.yml in the current working directory." + ) + sys.exit(1) + + log_file = sys.argv[1] + output = sys.argv[2] + + if not Path(log_file).exists(): + print(f"Log file not found: {log_file}") + sys.exit(1) + + rules = load_rules() + noisy, sensitive, severity = analyze(log_file, rules) + generate_html(noisy, sensitive, severity, output) + diff --git a/rules.yml b/rules.yml new file mode 100644 index 00000000..654b1d21 --- /dev/null +++ b/rules.yml @@ -0,0 +1,51 @@ +# ------------------------------ +# Rules for log analyzer +# ------------------------------ + +# Patterns to detect sensitive information +sensitive_patterns: + # JWT tokens + - '(?i)\btoken\s*[:=]\s*[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\b' + + # API keys + - '(?i)\b(api[_-]?key|apikey)\s*[:=]\s*[A-Za-z0-9]{20,}\b' + + # Password assignments + - '(?i)\b(password|passwd|pwd)\s*[:=]\s*\S{6,}\b' + + # Authorization headers + - '(?i)\bauthorization\s*[:=]\s*(bearer|basic)\s+[A-Za-z0-9+/=._-]+\b' + + # AWS keys + - '\bAKIA[0-9A-Z]{16}\b' + + # Private key blocks + - '-----BEGIN .*PRIVATE KEY-----' + + # URLs + - '\bhttps?://[^\s]+' + + # IP addresses + - '\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b' + + # MAC addresses + - '\b([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})\b' + +# Log levels considered noisy +noisy_log_levels: + - DEBUG + - TRACE + - INFO + +# Keywords indicating failure +failure_keywords: + - 'failed' + - 'error' + - 'exception' + - 'invalid' + - 'timeout' + +# Required log levels for failure messages +required_severity_on_failure: + - ERROR + - WARN diff --git a/src/rbus/rbus.c b/src/rbus/rbus.c index dfed4e50..ad72b045 100644 --- a/src/rbus/rbus.c +++ b/src/rbus/rbus.c @@ -2794,7 +2794,7 @@ static void _create_direct_connection_callback_handler (rbusHandle_t handle, rbu // PID is something unique but the same client can have direct connection to two different providers and that could lead to failure. // ex: if the consumer pid is 12345; - // Provider1 may open a private session on 127.0.0.1:12345 + // Provider1 may open a i private session on 127.0.0.1:12345 // Provider2 should not (could not) open private session on same address as 127.0.0.1:12345. // Provider 2 have to come up with some other logic