From 630c257d8768648f6ce476585c966fc8489518e7 Mon Sep 17 00:00:00 2001 From: Senthil Krishnamurthy Date: Tue, 23 Jun 2026 19:37:35 +0000 Subject: [PATCH 1/2] [ire_watchdog] Add DNX IRE data-path CRC watchdog daemons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Broadcom DNX/J3 ASICs, a single-bit error reported via the IRE_ErrorDataPathCrc interrupt can corrupt in-memory state (e.g. MACsec CKN) and lead to a downstream cascade — egress MACsec ACL flipping to DROP, LACP partner timeouts, eventually BGP %ADJCHANGE Down with reason "Interface down". Time-to-converge ~75s on observed incidents because LACP's 90s timeout was the fastest detector. This change adds two host-side systemd-managed daemons that detect the fault much earlier and fail-stop the affected DNX core's front-panel ports, dropping convergence to ~1-2s: * ire_watchdog (poll mode) Polls IRE_DATA_PATH_CRC_ERROR_COUNTER per-core at 1Hz via `docker exec syncd bcmcmd 'getreg ...'`. On increment above baseline, shuts all front-panel ports on the affected core via `sudo config interface shutdown`, logs CRITICAL, and latches. * ire_watchdog_syslog (syslog-tail mode, alternate) Tails /var/log/syslog for the SDK's `dnxc_interrupt_print_info: name=IRE_ErrorDataPathCrc` message. Same shutdown action. Zero ongoing syslog pollution since it doesn't invoke bcmcmd. Both are gated to switch_type == "voq" (DNX) via ExecCondition; the poll variant also exits cleanly if the register isn't recognized by the SDK (defense-in-depth for non-DNX images). Operators must explicitly `sudo config interface startup EthernetX` after reboot or syncd reload to recover — the shutdown is intentionally durable in CONFIG_DB to signal that manual review of the hardware is required. Both variants ship by default but only one should be enabled per DUT. Recommended: enable ire_watchdog_syslog (no syncd log pressure); use ire_watchdog if the SDK ever changes the dnxc_interrupt_print_info log format. Tested on Nexthop 5010 (DNX/J3, NH-5010-F-O64) — verified end-to-end that simulated counter increments on a specific core result in the correct set of front-panel ports being admin-down, with CRITICAL journal output, latching behavior, and clean recovery via manual startup. --- data/debian/rules | 2 + ...-services-data.ire-watchdog-syslog.service | 20 + ...ic-host-services-data.ire-watchdog.service | 20 + scripts/ire_watchdog | 364 ++++++++++++++++++ scripts/ire_watchdog_syslog | 260 +++++++++++++ setup.py | 4 +- 6 files changed, 669 insertions(+), 1 deletion(-) create mode 100644 data/debian/sonic-host-services-data.ire-watchdog-syslog.service create mode 100644 data/debian/sonic-host-services-data.ire-watchdog.service create mode 100755 scripts/ire_watchdog create mode 100755 scripts/ire_watchdog_syslog diff --git a/data/debian/rules b/data/debian/rules index f122c7c5..a01ac78d 100755 --- a/data/debian/rules +++ b/data/debian/rules @@ -25,5 +25,7 @@ override_dh_installsystemd: dh_installsystemd --no-start --name=console-monitor-dte dh_installsystemd --no-start --name=console-monitor-proxy@ dh_installsystemd --no-start --name=console-monitor-pty-bridge@ + dh_installsystemd --no-start --name=ire-watchdog + dh_installsystemd --no-start --name=ire-watchdog-syslog dh_installsystemd $(HOST_SERVICE_OPTS) --name=sonic-hostservice diff --git a/data/debian/sonic-host-services-data.ire-watchdog-syslog.service b/data/debian/sonic-host-services-data.ire-watchdog-syslog.service new file mode 100644 index 00000000..023fb467 --- /dev/null +++ b/data/debian/sonic-host-services-data.ire-watchdog-syslog.service @@ -0,0 +1,20 @@ +[Unit] +Description=IRE data-path CRC watchdog (syslog-tail mode) +Requires=rsyslog.service database.service +After=rsyslog.service database.service config-setup.service +BindsTo=sonic.target +After=sonic.target + +[Service] +Type=simple +# Gate to DNX/J3 platforms (switch_type == "voq"). The companion +# ire_watchdog.service (poll-register mode) and this unit are mutually +# exclusive — install/enable only one. +ExecCondition=/usr/bin/python3 -c "import sys; from sonic_py_common import device_info; sys.exit(0 if device_info.get_localhost_info('switch_type') == 'voq' else 1)" +ExecStart=/usr/local/bin/ire_watchdog_syslog +Restart=on-failure +RestartSec=10 +TimeoutStopSec=3 + +[Install] +WantedBy=sonic.target diff --git a/data/debian/sonic-host-services-data.ire-watchdog.service b/data/debian/sonic-host-services-data.ire-watchdog.service new file mode 100644 index 00000000..6b29b366 --- /dev/null +++ b/data/debian/sonic-host-services-data.ire-watchdog.service @@ -0,0 +1,20 @@ +[Unit] +Description=IRE data-path CRC watchdog (poll-register mode) +Requires=docker.service database.service +After=docker.service database.service config-setup.service +BindsTo=sonic.target +After=sonic.target + +[Service] +Type=simple +# Gate to DNX/J3 platforms (switch_type == "voq"). On non-DNX platforms the +# unit will not start. The daemon itself also exits cleanly if it detects +# IRE_DATA_PATH_CRC_ERROR_COUNTER isn't a known register on this ASIC. +ExecCondition=/usr/bin/python3 -c "import sys; from sonic_py_common import device_info; sys.exit(0 if device_info.get_localhost_info('switch_type') == 'voq' else 1)" +ExecStart=/usr/local/bin/ire_watchdog +Restart=on-failure +RestartSec=10 +TimeoutStopSec=3 + +[Install] +WantedBy=sonic.target diff --git a/scripts/ire_watchdog b/scripts/ire_watchdog new file mode 100755 index 00000000..f19ef9e3 --- /dev/null +++ b/scripts/ire_watchdog @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +""" +ire_watchdog — IRE data-path CRC error watchdog (poll-register mode). + +Polls IRE_DATA_PATH_CRC_ERROR_COUNTER on each DNX core once per second via +`docker exec syncd bcmcmd 'getreg ...'`. If any per-core counter increments +above the startup baseline, shuts down all front-panel ports on that core +via `sudo config interface shutdown`, raises a CRITICAL syslog, and latches +(no auto-recovery). + +Design intent: treat any IRE data-path CRC error as a hard fail-stop event. +The SDK already masks the interrupt after a single failed-clear-attempt +(see syslog `dnxc_intr_unmask_handling_in_isr: ... exceed threshold 1!!! Masked`). +Memory corruption persists in chip SRAM until reset, so manual operator +recovery (reboot or syncd reload) is required. + +Runs in the host namespace. Uses `docker exec syncd bcmcmd ...` to read the +register because the bcm shell socket lives only inside the syncd container. +Shuts down ports via `sudo config interface shutdown` so the state persists +in CONFIG_DB across reboots until the operator explicitly recovers. + +If syncd is not running (container down or restarting), the daemon logs at +WARNING and continues polling — this is a known transient state during +syncd reload/swss restart and must not crash the watchdog. + +This daemon is DNX-specific. The accompanying systemd unit gates activation +on switch_type == "voq" via ExecCondition; additionally, the daemon exits +cleanly at startup if it detects the IRE_DATA_PATH_CRC_ERROR_COUNTER +register is unknown to the SDK (a fast-path safety belt for non-DNX images +that somehow start the unit). + +Note: every `bcmcmd` invocation causes syncd to emit two WARNING lines from +`sai_driver_shell:367`. To suppress this 1Hz syslog noise, install an +rsyslog filter such as: + + if ($programname startswith "syncd" and $msg contains "BCM shell command:") then stop + +A companion syslog-tail variant (ire_watchdog_syslog) avoids this noise +entirely by watching for the SDK's dnxc_interrupt_print_info message instead +of polling, at the cost of a coupling to the log message format. + +Recovery: not implemented. Operator runbook: + 1. Capture techsupport. + 2. Reload syncd (or reboot the device). + 3. Verify counter is clear: docker exec syncd bcmcmd 'getreg IRE_DATA_PATH_CRC_ERROR_COUNTER' + 4. Manually bring ports back up: sudo config interface startup EthernetX +""" + +import argparse +import logging +import re +import subprocess +import sys +import time +from pathlib import Path + +PORT_CONFIG_GLOB = '/usr/share/sonic/device/*/*/port_config.ini' +SYNCD_CONTAINER = 'syncd' +BCMCMD = ['docker', 'exec', SYNCD_CONTAINER, 'bcmcmd'] +COUNTER_REG = 'IRE_DATA_PATH_CRC_ERROR_COUNTER' +POLL_INTERVAL_SEC = 1.0 +BCMCMD_TIMEOUT_SEC = 5.0 +SHUTDOWN_CMD = ['sudo', 'config', 'interface', 'shutdown'] + +# IRE_DATA_PATH_CRC_ERROR_COUNTER.IRE3[0x169]=42: +# Value before the colon may be decimal (e.g. =0) or hex (e.g. =0x42); accept either. +LINE_RE = re.compile( + r'IRE_DATA_PATH_CRC_ERROR_COUNTER\.IRE(\d+)\[[^\]]+\]=(0x[0-9a-fA-F]+|\d+)' +) + + +class SyncdUnavailable(Exception): + """syncd container is not running or otherwise unreachable.""" + + +class PlatformNotSupported(Exception): + """ASIC doesn't expose IRE_DATA_PATH_CRC_ERROR_COUNTER (i.e. not DNX).""" + + +def _parse_int(s: str) -> int: + return int(s, 16) if s.startswith('0x') else int(s) + + +def setup_logging(verbose: bool) -> None: + """Log to stderr only. + + When run under systemd (the production case), the unit file declares + ``StandardError=journal`` and ``SyslogIdentifier=ire_watchdog``, so + everything written to stderr is captured by journald, tagged correctly, + and forwarded to /var/log/syslog by rsyslog. A direct SysLogHandler + would cause messages to be logged twice with mangled identifiers + (``ire_watchdogire_watchdog[PID]:``), so we don't use one. + + When run interactively (e.g. ``--dry-run --simulate-core``), stderr + output goes to the terminal as expected. + """ + root = logging.getLogger() + root.setLevel(logging.DEBUG if verbose else logging.INFO) + fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + sh = logging.StreamHandler(sys.stderr) + sh.setFormatter(fmt) + root.addHandler(sh) + + +def find_port_config() -> Path: + """Locate port_config.ini under the running hwsku.""" + try: + hwsku = subprocess.run( + ['sonic-cfggen', '-d', '-v', 'DEVICE_METADATA.localhost.hwsku'], + capture_output=True, text=True, check=True, timeout=5, + ).stdout.strip() + platform = subprocess.run( + ['sonic-cfggen', '-d', '-v', 'DEVICE_METADATA.localhost.platform'], + capture_output=True, text=True, check=True, timeout=5, + ).stdout.strip() + path = Path(f'/usr/share/sonic/device/{platform}/{hwsku}/port_config.ini') + if path.exists(): + return path + except Exception as e: + logging.warning(f'sonic-cfggen probe failed: {e}, falling back to glob') + + matches = sorted(Path('/').glob(PORT_CONFIG_GLOB.lstrip('/'))) + if not matches: + raise RuntimeError(f'no port_config.ini found under {PORT_CONFIG_GLOB}') + return matches[0] + + +def build_port_to_core(port_config: Path) -> dict[str, int]: + """Parse port_config.ini and return {EthernetX: core_id}.""" + port_to_core: dict[str, int] = {} + header: list[str] | None = None + for line in port_config.read_text().splitlines(): + line = line.strip() + if not line or (line.startswith('#') and 'name' not in line): + continue + if line.startswith('#'): + header = line.lstrip('#').split() + continue + if header is None: + raise RuntimeError(f'no header in {port_config}') + fields = line.split() + if len(fields) < len(header): + continue + row = dict(zip(header, fields)) + name = row.get('name') + core_id = row.get('core_id') + if name and core_id and core_id.isdigit(): + port_to_core[name] = int(core_id) + if not port_to_core: + raise RuntimeError(f'no port->core mappings parsed from {port_config}') + return port_to_core + + +def _is_syncd_unavailable_error(stderr: str) -> bool: + """Heuristics for stderr indicating syncd container isn't usable.""" + s = stderr.lower() + return ( + 'is not running' in s + or 'no such container' in s + or 'error response from daemon' in s + or 'cannot connect to the docker daemon' in s + ) + + +def _is_unknown_register_output(output: str) -> bool: + """Heuristics for bcmcmd output indicating the register doesn't exist + on this ASIC (non-DNX platform). The bcm shell typically responds with + 'Symbol not found' or similar when given an unknown register name.""" + s = output.lower() + return ( + 'symbol not found' in s + or 'symbol unknown' in s + or 'unknown symbol' in s + or 'no such symbol' in s + or 'not a valid register' in s + ) + + +def syncd_running() -> bool: + """Quick check: is the syncd container in the running state?""" + try: + r = subprocess.run( + ['docker', 'inspect', '--format', '{{.State.Running}}', SYNCD_CONTAINER], + capture_output=True, text=True, timeout=5, + ) + return r.returncode == 0 and r.stdout.strip() == 'true' + except Exception: + return False + + +def read_counters() -> dict[int, int]: + """Read IRE_DATA_PATH_CRC_ERROR_COUNTER and return {core_id: count}. + + Raises SyncdUnavailable if syncd is not running. + Raises RuntimeError on other bcmcmd/parsing failures. + """ + try: + out = subprocess.run( + BCMCMD + [f'getreg {COUNTER_REG}'], + capture_output=True, text=True, timeout=BCMCMD_TIMEOUT_SEC, + ) + except subprocess.TimeoutExpired as e: + raise RuntimeError(f'bcmcmd timed out after {BCMCMD_TIMEOUT_SEC}s') from e + except FileNotFoundError as e: + raise SyncdUnavailable(f'docker binary not found: {e}') + + if out.returncode != 0: + stderr = out.stderr.strip() + if _is_syncd_unavailable_error(stderr) or not syncd_running(): + raise SyncdUnavailable(stderr or 'syncd container not running') + raise RuntimeError(f'bcmcmd failed (rc={out.returncode}): {stderr}') + + # On non-DNX platforms, bcmcmd accepts the command but reports an unknown + # symbol. Treat as "platform not supported" and let the caller exit clean. + if _is_unknown_register_output(out.stdout) or _is_unknown_register_output(out.stderr): + raise PlatformNotSupported( + f'{COUNTER_REG} is not a valid register on this ASIC' + ) + + counters: dict[int, int] = {} + for line in out.stdout.splitlines(): + m = LINE_RE.search(line) + if m: + counters[int(m.group(1))] = _parse_int(m.group(2)) + if not counters: + # Empty/garbage output sometimes seen during syncd start before bcm + # shell is ready. Distinguish from "syncd is fine but parser broke". + if not syncd_running(): + raise SyncdUnavailable('syncd not running during read') + raise RuntimeError(f'no counters parsed from bcmcmd output: {out.stdout!r}') + return counters + + +def shut_ports(ports: list[str]) -> list[tuple[str, bool, str]]: + """Shut down ports via `sudo config interface shutdown`. + + Persists admin-down state in config_db across reboots — operator must + explicitly run `sudo config interface startup EthernetX` to revert. + + Returns [(port, ok, msg)]. + """ + results = [] + for port in ports: + try: + r = subprocess.run( + SHUTDOWN_CMD + [port], + capture_output=True, text=True, timeout=15, + ) + msg = (r.stderr.strip() or r.stdout.strip()) + results.append((port, r.returncode == 0, msg)) + except subprocess.TimeoutExpired: + results.append((port, False, 'timeout')) + except Exception as e: + results.append((port, False, repr(e))) + return results + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + p.add_argument('--interval', type=float, default=POLL_INTERVAL_SEC, + help='polling interval in seconds (default: 1.0)') + p.add_argument('--dry-run', action='store_true', + help='log what would be shut down but do not execute') + p.add_argument('--verbose', action='store_true', + help='enable debug logging') + p.add_argument('--simulate-core', type=int, default=None, metavar='N', + help='for testing: force baseline of core N to -1 so the next read' + ' looks like an increment (use with --dry-run)') + return p.parse_args() + + +def main() -> int: + args = parse_args() + setup_logging(args.verbose) + + log = logging.getLogger() + log.info(f'starting ire_watchdog (poll mode, interval={args.interval}s, dry_run={args.dry_run})') + + port_config = find_port_config() + log.info(f'port config: {port_config}') + port_to_core = build_port_to_core(port_config) + core_to_ports: dict[int, list[str]] = {} + for port, core in port_to_core.items(): + core_to_ports.setdefault(core, []).append(port) + for core, ports in sorted(core_to_ports.items()): + ports.sort(key=lambda p: int(p.removeprefix('Ethernet')) if p.removeprefix('Ethernet').isdigit() else 9999) + log.info(f'core {core}: {len(ports)} ports ({ports[0]}..{ports[-1]})') + + # Establish baseline. The counter can be non-zero at boot (e.g. SDK init); + # we only act on increments above this baseline. If syncd isn't running + # yet (e.g. daemon started before swss.service), wait for it. If the + # register doesn't exist on this ASIC (non-DNX), exit cleanly so systemd + # marks the unit as inactive rather than failing. + baseline: dict[int, int] | None = None + while baseline is None: + try: + baseline = read_counters() + except PlatformNotSupported as e: + log.info(f'platform does not support IRE CRC counter, exiting: {e}') + return 0 + except SyncdUnavailable as e: + log.warning(f'syncd not available, retrying baseline read in 5s: {e}') + time.sleep(5) + except Exception as e: + log.error(f'failed to read baseline: {e}; retrying in 5s') + time.sleep(5) + log.info(f'baseline counters: {baseline}') + + if args.simulate_core is not None: + baseline[args.simulate_core] = -1 + log.warning(f'SIMULATION: baseline[core {args.simulate_core}] set to -1; ' + f'next read will trigger detection') + + latched_cores: set[int] = set() + consecutive_errors = 0 + syncd_was_down = False + + while True: + try: + current = read_counters() + if syncd_was_down: + log.info('syncd is back, resuming polling') + syncd_was_down = False + consecutive_errors = 0 + except SyncdUnavailable as e: + if not syncd_was_down: + log.warning(f'syncd unavailable, polling will resume when it returns: {e}') + syncd_was_down = True + time.sleep(args.interval) + continue + except Exception as e: + consecutive_errors += 1 + level = logging.WARNING if consecutive_errors < 5 else logging.ERROR + log.log(level, f'read failed ({consecutive_errors}x): {e}') + time.sleep(args.interval) + continue + + for core, count in sorted(current.items()): + base = baseline.get(core, 0) + if count > base and core not in latched_cores: + ports = sorted(core_to_ports.get(core, []), + key=lambda p: int(p.removeprefix('Ethernet')) if p.removeprefix('Ethernet').isdigit() else 9999) + log.critical( + f'IRE_DATA_PATH_CRC_ERROR_COUNTER.IRE{core} ' + f'increment detected: baseline={base} current={count}; ' + f'shutting {len(ports)} ports on core {core}: {ports}; ' + f'manual operator recovery required (reload syncd/reboot)' + ) + if args.dry_run: + log.warning(f'DRY RUN: would shut {ports}') + else: + results = shut_ports(ports) + for port, ok, msg in results: + if ok: + log.info(f' shut {port}: OK') + else: + log.error(f' shut {port}: FAILED: {msg}') + latched_cores.add(core) + + time.sleep(args.interval) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/ire_watchdog_syslog b/scripts/ire_watchdog_syslog new file mode 100755 index 00000000..185b39f3 --- /dev/null +++ b/scripts/ire_watchdog_syslog @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +""" +ire_watchdog.py — IRE data-path CRC interrupt watchdog (syslog-tail mode). + +Watches /var/log/syslog for the SDK's `dnxc_interrupt_print_info` message +emitted on IRE_ErrorDataPathCrc. When seen, shuts down all front-panel +ports on the affected core via `sudo config interface shutdown`, raises a +CRITICAL syslog, and latches (no auto-recovery). + +Design intent: treat any IRE data-path CRC error as a hard fail-stop event. +The SDK already masks the interrupt after a single failed clear attempt, +so the message is guaranteed to fire exactly once per occurrence (see +production evidence: `dnxc_intr_unmask_handling_in_isr: ... exceed threshold +1!!! Masked`). Memory corruption persists in chip SRAM until reset, so +manual operator recovery (reboot or syncd reload) is required. + +Why syslog-tail instead of bcmcmd polling: every `bcmcmd` invocation makes +syncd log a WARNING from sai_driver_shell:367, polluting /var/log/syslog at +1Hz. Watching the existing dnxc_interrupt_print_info message that the SDK +already emits on the actual fault produces zero ongoing log noise and +catches the event with sub-100ms latency. + +Runs in the host (DUT) namespace. Reads /var/log/syslog via `tail -F`, +which handles log rotation, truncation, and rsyslog restart transparently. + +Production sample line (WAW23 dump, 2026-06-09 incident): + INFO syncd#supervisord: syncd 0:dnxc_interrupt_print_info: + name=IRE_ErrorDataPathCrc, id=1666, index=0, block=3, unit=0, + recurring_action=0 | Check Configuration | None + +Recovery: not implemented. Operator runbook: + 1. Capture techsupport. + 2. Reload syncd (or reboot the device). + 3. Manually bring ports back up: sudo config interface startup EthernetX +""" + +import argparse +import logging +import logging.handlers +import re +import subprocess +import sys +import time +from pathlib import Path + +PORT_CONFIG_GLOB = '/usr/share/sonic/device/*/*/port_config.ini' +SYSLOG_PATH = '/var/log/syslog' +SHUTDOWN_CMD = ['sudo', 'config', 'interface', 'shutdown'] + +# Match dnxc_interrupt_print_info lines for the IRE data-path CRC error. +# We extract the `block=N` field which maps 1:1 to DNX core N. +# Order of fields is stable in the SDK (name first, then id/index/block/unit), +# but we match permissively in case future SDK versions reorder. +INTR_RE = re.compile( + r'dnxc_interrupt_print_info:.*?name=IRE_ErrorDataPathCrc.*?block=(\d+)' +) + + +class WatchdogError(Exception): + """Recoverable error during normal operation.""" + + +def setup_logging(verbose: bool) -> None: + root = logging.getLogger() + root.setLevel(logging.DEBUG if verbose else logging.INFO) + fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + + sh = logging.StreamHandler(sys.stderr) + sh.setFormatter(fmt) + root.addHandler(sh) + + try: + syslog = logging.handlers.SysLogHandler(address='/dev/log') + syslog.ident = 'ire_watchdog' + syslog.setFormatter(logging.Formatter('ire_watchdog: %(message)s')) + root.addHandler(syslog) + except Exception as e: + root.warning(f'syslog handler unavailable: {e}') + + +def find_port_config() -> Path: + """Locate port_config.ini under the running hwsku.""" + try: + hwsku = subprocess.run( + ['sonic-cfggen', '-d', '-v', 'DEVICE_METADATA.localhost.hwsku'], + capture_output=True, text=True, check=True, timeout=5, + ).stdout.strip() + platform = subprocess.run( + ['sonic-cfggen', '-d', '-v', 'DEVICE_METADATA.localhost.platform'], + capture_output=True, text=True, check=True, timeout=5, + ).stdout.strip() + path = Path(f'/usr/share/sonic/device/{platform}/{hwsku}/port_config.ini') + if path.exists(): + return path + except Exception as e: + logging.warning(f'sonic-cfggen probe failed: {e}, falling back to glob') + + matches = sorted(Path('/').glob(PORT_CONFIG_GLOB.lstrip('/'))) + if not matches: + raise RuntimeError(f'no port_config.ini found under {PORT_CONFIG_GLOB}') + return matches[0] + + +def build_port_to_core(port_config: Path) -> dict[str, int]: + """Parse port_config.ini and return {EthernetX: core_id}.""" + port_to_core: dict[str, int] = {} + header: list[str] | None = None + for line in port_config.read_text().splitlines(): + line = line.strip() + if not line or (line.startswith('#') and 'name' not in line): + continue + if line.startswith('#'): + header = line.lstrip('#').split() + continue + if header is None: + raise RuntimeError(f'no header in {port_config}') + fields = line.split() + if len(fields) < len(header): + continue + row = dict(zip(header, fields)) + name = row.get('name') + core_id = row.get('core_id') + if name and core_id and core_id.isdigit(): + port_to_core[name] = int(core_id) + if not port_to_core: + raise RuntimeError(f'no port->core mappings parsed from {port_config}') + return port_to_core + + +def shut_ports(ports: list[str]) -> list[tuple[str, bool, str]]: + """Shut down ports via `sudo config interface shutdown`. + + Persists admin-down state in config_db across reboots — operator must + explicitly run `sudo config interface startup EthernetX` to revert. + + Returns [(port, ok, msg)]. + """ + results = [] + for port in ports: + try: + r = subprocess.run( + SHUTDOWN_CMD + [port], + capture_output=True, text=True, timeout=15, + ) + msg = (r.stderr.strip() or r.stdout.strip()) + results.append((port, r.returncode == 0, msg)) + except subprocess.TimeoutExpired: + results.append((port, False, 'timeout')) + except Exception as e: + results.append((port, False, repr(e))) + return results + + +def open_syslog_tail() -> subprocess.Popen: + """Spawn `tail -F` on /var/log/syslog. + + `tail -F` (capital F) follows the file by name — it re-opens after log + rotation, truncation, or rsyslog restart. -n0 starts at end-of-file so + we only see new lines. + """ + return subprocess.Popen( + ['tail', '-n0', '-F', SYSLOG_PATH], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # line-buffered + ) + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + p.add_argument('--dry-run', action='store_true', + help='log what would be shut down but do not execute') + p.add_argument('--verbose', action='store_true', + help='enable debug logging') + p.add_argument('--simulate-line', metavar='LINE', + help='for testing: inject a fake syslog line into the ' + 'detection path before tailing real syslog') + return p.parse_args() + + +def handle_match(core: int, ports_on_core: list[str], dry_run: bool, log: logging.Logger, + latched_cores: set[int]) -> None: + if core in latched_cores: + log.debug(f'core {core} already latched, ignoring duplicate event') + return + + log.critical( + f'IRE_ErrorDataPathCrc on block/core {core}; shutting ' + f'{len(ports_on_core)} ports: {ports_on_core}; manual operator ' + f'recovery required (reload syncd or reboot)' + ) + if dry_run: + log.warning(f'DRY RUN: would shut {ports_on_core}') + else: + for port, ok, msg in shut_ports(ports_on_core): + if ok: + log.info(f' shut {port}: OK') + else: + log.error(f' shut {port}: FAILED: {msg}') + latched_cores.add(core) + + +def main() -> int: + args = parse_args() + setup_logging(args.verbose) + log = logging.getLogger() + log.info(f'starting ire_watchdog (syslog-tail mode, dry_run={args.dry_run})') + + port_config = find_port_config() + log.info(f'port config: {port_config}') + port_to_core = build_port_to_core(port_config) + core_to_ports: dict[int, list[str]] = {} + for port, core in port_to_core.items(): + core_to_ports.setdefault(core, []).append(port) + for core, ports in sorted(core_to_ports.items()): + ports.sort(key=lambda p: int(p.removeprefix('Ethernet')) if p.removeprefix('Ethernet').isdigit() else 9999) + log.info(f'core {core}: {len(ports)} ports ({ports[0]}..{ports[-1]})') + + latched_cores: set[int] = set() + + # Optional injection for testing without touching real syslog + if args.simulate_line: + log.warning(f'SIMULATION: injecting line: {args.simulate_line}') + m = INTR_RE.search(args.simulate_line) + if m: + core = int(m.group(1)) + ports = core_to_ports.get(core, []) + handle_match(core, ports, args.dry_run, log, latched_cores) + else: + log.error(f'SIMULATION: line did not match regex') + + # Main loop: tail syslog, restart `tail` on EOF or error + while True: + try: + log.info(f'tailing {SYSLOG_PATH}') + proc = open_syslog_tail() + assert proc.stdout is not None + for line in proc.stdout: + m = INTR_RE.search(line) + if not m: + continue + core = int(m.group(1)) + log.info(f'matched interrupt line for core {core}: {line.strip()}') + ports = core_to_ports.get(core, []) + if not ports: + log.error(f'no ports mapped to core {core}; cannot act') + continue + handle_match(core, ports, args.dry_run, log, latched_cores) + # tail's stdout EOF — shouldn't happen with -F, but reconnect anyway + log.warning(f'tail -F exited (rc={proc.poll()}); restarting in 2s') + except Exception as e: + log.error(f'tail loop error: {e}; restarting in 5s') + time.sleep(5) + continue + time.sleep(2) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setup.py b/setup.py index c8aa6bf3..ec7c9915 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,9 @@ 'scripts/gnoi_shutdown_daemon.py', 'scripts/sonic-host-server', 'scripts/ldap.py', - 'scripts/console-monitor' + 'scripts/console-monitor', + 'scripts/ire_watchdog', + 'scripts/ire_watchdog_syslog' ], install_requires = [ 'dbus-python', From 82e44c7668ba4d6b056b340adf93b5b6ecbecb88 Mon Sep 17 00:00:00 2001 From: Senthil Krishnamurthy Date: Tue, 23 Jun 2026 19:42:00 +0000 Subject: [PATCH 2/2] [ire_watchdog] Drop syslog-tail variant Ship only the poll-register variant. The syslog-tail companion (ire_watchdog_syslog) is removed pending a separate discussion about which detection mechanism upstream prefers. The polling daemon is the more robust of the two (no coupling to SDK log-message format, unaffected by rsyslog issues) and is the variant validated on the production hardware where the original incident was observed. Removed: * scripts/ire_watchdog_syslog * data/debian/sonic-host-services-data.ire-watchdog-syslog.service * setup.py entry for the syslog variant * data/debian/rules dh_installsystemd entry for ire-watchdog-syslog * Reference in scripts/ire_watchdog docstring to the companion variant --- data/debian/rules | 1 - ...-services-data.ire-watchdog-syslog.service | 20 -- scripts/ire_watchdog | 104 +++---- scripts/ire_watchdog_syslog | 260 ------------------ setup.py | 3 +- 5 files changed, 43 insertions(+), 345 deletions(-) delete mode 100644 data/debian/sonic-host-services-data.ire-watchdog-syslog.service delete mode 100755 scripts/ire_watchdog_syslog diff --git a/data/debian/rules b/data/debian/rules index a01ac78d..e8224fda 100755 --- a/data/debian/rules +++ b/data/debian/rules @@ -26,6 +26,5 @@ override_dh_installsystemd: dh_installsystemd --no-start --name=console-monitor-proxy@ dh_installsystemd --no-start --name=console-monitor-pty-bridge@ dh_installsystemd --no-start --name=ire-watchdog - dh_installsystemd --no-start --name=ire-watchdog-syslog dh_installsystemd $(HOST_SERVICE_OPTS) --name=sonic-hostservice diff --git a/data/debian/sonic-host-services-data.ire-watchdog-syslog.service b/data/debian/sonic-host-services-data.ire-watchdog-syslog.service deleted file mode 100644 index 023fb467..00000000 --- a/data/debian/sonic-host-services-data.ire-watchdog-syslog.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=IRE data-path CRC watchdog (syslog-tail mode) -Requires=rsyslog.service database.service -After=rsyslog.service database.service config-setup.service -BindsTo=sonic.target -After=sonic.target - -[Service] -Type=simple -# Gate to DNX/J3 platforms (switch_type == "voq"). The companion -# ire_watchdog.service (poll-register mode) and this unit are mutually -# exclusive — install/enable only one. -ExecCondition=/usr/bin/python3 -c "import sys; from sonic_py_common import device_info; sys.exit(0 if device_info.get_localhost_info('switch_type') == 'voq' else 1)" -ExecStart=/usr/local/bin/ire_watchdog_syslog -Restart=on-failure -RestartSec=10 -TimeoutStopSec=3 - -[Install] -WantedBy=sonic.target diff --git a/scripts/ire_watchdog b/scripts/ire_watchdog index f19ef9e3..20c7650f 100755 --- a/scripts/ire_watchdog +++ b/scripts/ire_watchdog @@ -1,51 +1,4 @@ #!/usr/bin/env python3 -""" -ire_watchdog — IRE data-path CRC error watchdog (poll-register mode). - -Polls IRE_DATA_PATH_CRC_ERROR_COUNTER on each DNX core once per second via -`docker exec syncd bcmcmd 'getreg ...'`. If any per-core counter increments -above the startup baseline, shuts down all front-panel ports on that core -via `sudo config interface shutdown`, raises a CRITICAL syslog, and latches -(no auto-recovery). - -Design intent: treat any IRE data-path CRC error as a hard fail-stop event. -The SDK already masks the interrupt after a single failed-clear-attempt -(see syslog `dnxc_intr_unmask_handling_in_isr: ... exceed threshold 1!!! Masked`). -Memory corruption persists in chip SRAM until reset, so manual operator -recovery (reboot or syncd reload) is required. - -Runs in the host namespace. Uses `docker exec syncd bcmcmd ...` to read the -register because the bcm shell socket lives only inside the syncd container. -Shuts down ports via `sudo config interface shutdown` so the state persists -in CONFIG_DB across reboots until the operator explicitly recovers. - -If syncd is not running (container down or restarting), the daemon logs at -WARNING and continues polling — this is a known transient state during -syncd reload/swss restart and must not crash the watchdog. - -This daemon is DNX-specific. The accompanying systemd unit gates activation -on switch_type == "voq" via ExecCondition; additionally, the daemon exits -cleanly at startup if it detects the IRE_DATA_PATH_CRC_ERROR_COUNTER -register is unknown to the SDK (a fast-path safety belt for non-DNX images -that somehow start the unit). - -Note: every `bcmcmd` invocation causes syncd to emit two WARNING lines from -`sai_driver_shell:367`. To suppress this 1Hz syslog noise, install an -rsyslog filter such as: - - if ($programname startswith "syncd" and $msg contains "BCM shell command:") then stop - -A companion syslog-tail variant (ire_watchdog_syslog) avoids this noise -entirely by watching for the SDK's dnxc_interrupt_print_info message instead -of polling, at the cost of a coupling to the log message format. - -Recovery: not implemented. Operator runbook: - 1. Capture techsupport. - 2. Reload syncd (or reboot the device). - 3. Verify counter is clear: docker exec syncd bcmcmd 'getreg IRE_DATA_PATH_CRC_ERROR_COUNTER' - 4. Manually bring ports back up: sudo config interface startup EthernetX -""" - import argparse import logging import re @@ -91,8 +44,8 @@ def setup_logging(verbose: bool) -> None: would cause messages to be logged twice with mangled identifiers (``ire_watchdogire_watchdog[PID]:``), so we don't use one. - When run interactively (e.g. ``--dry-run --simulate-core``), stderr - output goes to the terminal as expected. + When run interactively (e.g. ``--dry-run``), stderr output goes to + the terminal as expected. """ root = logging.getLogger() root.setLevel(logging.DEBUG if verbose else logging.INFO) @@ -188,11 +141,36 @@ def syncd_running() -> bool: return False +def _reap_leaked_bcmcmd() -> None: + """Kill bcmcmd processes left inside syncd by previous timeouts. + + `docker exec` does NOT propagate SIGKILL into the container, so when + `subprocess.run(timeout=...)` fires we leak one bcmcmd process per call. + Worse, if the bcm shell is in a non-default mode (e.g. cint) every + queued `getreg` blocks until a human exits that mode, so the orphan + count grows monotonically. + + Pattern-match on our specific command line so we only reap our own + orphans. An interactive `bcmcmd` session a human may have open shows + up as bare `bcmcmd` (no args) and won't match. + """ + try: + subprocess.run( + ['docker', 'exec', SYNCD_CONTAINER, 'pkill', '-9', '-f', + f'bcmcmd.*getreg.*{COUNTER_REG}'], + capture_output=True, timeout=5, + ) + except Exception: + # Best-effort cleanup — never let reap failures crash the daemon. + pass + + def read_counters() -> dict[int, int]: """Read IRE_DATA_PATH_CRC_ERROR_COUNTER and return {core_id: count}. Raises SyncdUnavailable if syncd is not running. - Raises RuntimeError on other bcmcmd/parsing failures. + Raises RuntimeError on other bcmcmd/parsing failures (including + timeouts; the orphan bcmcmd inside syncd is reaped before re-raising). """ try: out = subprocess.run( @@ -200,6 +178,7 @@ def read_counters() -> dict[int, int]: capture_output=True, text=True, timeout=BCMCMD_TIMEOUT_SEC, ) except subprocess.TimeoutExpired as e: + _reap_leaked_bcmcmd() raise RuntimeError(f'bcmcmd timed out after {BCMCMD_TIMEOUT_SEC}s') from e except FileNotFoundError as e: raise SyncdUnavailable(f'docker binary not found: {e}') @@ -256,16 +235,15 @@ def shut_ports(ports: list[str]) -> list[tuple[str, bool, str]]: def parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + p = argparse.ArgumentParser( + description='IRE data-path CRC error watchdog (poll-register mode)' + ) p.add_argument('--interval', type=float, default=POLL_INTERVAL_SEC, help='polling interval in seconds (default: 1.0)') p.add_argument('--dry-run', action='store_true', help='log what would be shut down but do not execute') p.add_argument('--verbose', action='store_true', help='enable debug logging') - p.add_argument('--simulate-core', type=int, default=None, metavar='N', - help='for testing: force baseline of core N to -1 so the next read' - ' looks like an increment (use with --dry-run)') return p.parse_args() @@ -306,11 +284,6 @@ def main() -> int: time.sleep(5) log.info(f'baseline counters: {baseline}') - if args.simulate_core is not None: - baseline[args.simulate_core] = -1 - log.warning(f'SIMULATION: baseline[core {args.simulate_core}] set to -1; ' - f'next read will trigger detection') - latched_cores: set[int] = set() consecutive_errors = 0 syncd_was_down = False @@ -331,8 +304,15 @@ def main() -> int: except Exception as e: consecutive_errors += 1 level = logging.WARNING if consecutive_errors < 5 else logging.ERROR - log.log(level, f'read failed ({consecutive_errors}x): {e}') - time.sleep(args.interval) + # Exponential backoff: 1s, 2s, 4s, 8s, ..., capped at 60s. Keeps + # us responsive to transient blips but stops piling on if the + # bcm shell is wedged in cint mode or similar — every queued + # bcmcmd inside syncd is another orphan to reap. + backoff = min(args.interval * (2 ** min(consecutive_errors - 1, 6)), + 60.0) + log.log(level, f'read failed ({consecutive_errors}x): {e}; ' + f'sleeping {backoff:.1f}s before retry') + time.sleep(backoff) continue for core, count in sorted(current.items()): @@ -344,7 +324,7 @@ def main() -> int: f'IRE_DATA_PATH_CRC_ERROR_COUNTER.IRE{core} ' f'increment detected: baseline={base} current={count}; ' f'shutting {len(ports)} ports on core {core}: {ports}; ' - f'manual operator recovery required (reload syncd/reboot)' + f'manual operator recovery required' ) if args.dry_run: log.warning(f'DRY RUN: would shut {ports}') diff --git a/scripts/ire_watchdog_syslog b/scripts/ire_watchdog_syslog deleted file mode 100755 index 185b39f3..00000000 --- a/scripts/ire_watchdog_syslog +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env python3 -""" -ire_watchdog.py — IRE data-path CRC interrupt watchdog (syslog-tail mode). - -Watches /var/log/syslog for the SDK's `dnxc_interrupt_print_info` message -emitted on IRE_ErrorDataPathCrc. When seen, shuts down all front-panel -ports on the affected core via `sudo config interface shutdown`, raises a -CRITICAL syslog, and latches (no auto-recovery). - -Design intent: treat any IRE data-path CRC error as a hard fail-stop event. -The SDK already masks the interrupt after a single failed clear attempt, -so the message is guaranteed to fire exactly once per occurrence (see -production evidence: `dnxc_intr_unmask_handling_in_isr: ... exceed threshold -1!!! Masked`). Memory corruption persists in chip SRAM until reset, so -manual operator recovery (reboot or syncd reload) is required. - -Why syslog-tail instead of bcmcmd polling: every `bcmcmd` invocation makes -syncd log a WARNING from sai_driver_shell:367, polluting /var/log/syslog at -1Hz. Watching the existing dnxc_interrupt_print_info message that the SDK -already emits on the actual fault produces zero ongoing log noise and -catches the event with sub-100ms latency. - -Runs in the host (DUT) namespace. Reads /var/log/syslog via `tail -F`, -which handles log rotation, truncation, and rsyslog restart transparently. - -Production sample line (WAW23 dump, 2026-06-09 incident): - INFO syncd#supervisord: syncd 0:dnxc_interrupt_print_info: - name=IRE_ErrorDataPathCrc, id=1666, index=0, block=3, unit=0, - recurring_action=0 | Check Configuration | None - -Recovery: not implemented. Operator runbook: - 1. Capture techsupport. - 2. Reload syncd (or reboot the device). - 3. Manually bring ports back up: sudo config interface startup EthernetX -""" - -import argparse -import logging -import logging.handlers -import re -import subprocess -import sys -import time -from pathlib import Path - -PORT_CONFIG_GLOB = '/usr/share/sonic/device/*/*/port_config.ini' -SYSLOG_PATH = '/var/log/syslog' -SHUTDOWN_CMD = ['sudo', 'config', 'interface', 'shutdown'] - -# Match dnxc_interrupt_print_info lines for the IRE data-path CRC error. -# We extract the `block=N` field which maps 1:1 to DNX core N. -# Order of fields is stable in the SDK (name first, then id/index/block/unit), -# but we match permissively in case future SDK versions reorder. -INTR_RE = re.compile( - r'dnxc_interrupt_print_info:.*?name=IRE_ErrorDataPathCrc.*?block=(\d+)' -) - - -class WatchdogError(Exception): - """Recoverable error during normal operation.""" - - -def setup_logging(verbose: bool) -> None: - root = logging.getLogger() - root.setLevel(logging.DEBUG if verbose else logging.INFO) - fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - - sh = logging.StreamHandler(sys.stderr) - sh.setFormatter(fmt) - root.addHandler(sh) - - try: - syslog = logging.handlers.SysLogHandler(address='/dev/log') - syslog.ident = 'ire_watchdog' - syslog.setFormatter(logging.Formatter('ire_watchdog: %(message)s')) - root.addHandler(syslog) - except Exception as e: - root.warning(f'syslog handler unavailable: {e}') - - -def find_port_config() -> Path: - """Locate port_config.ini under the running hwsku.""" - try: - hwsku = subprocess.run( - ['sonic-cfggen', '-d', '-v', 'DEVICE_METADATA.localhost.hwsku'], - capture_output=True, text=True, check=True, timeout=5, - ).stdout.strip() - platform = subprocess.run( - ['sonic-cfggen', '-d', '-v', 'DEVICE_METADATA.localhost.platform'], - capture_output=True, text=True, check=True, timeout=5, - ).stdout.strip() - path = Path(f'/usr/share/sonic/device/{platform}/{hwsku}/port_config.ini') - if path.exists(): - return path - except Exception as e: - logging.warning(f'sonic-cfggen probe failed: {e}, falling back to glob') - - matches = sorted(Path('/').glob(PORT_CONFIG_GLOB.lstrip('/'))) - if not matches: - raise RuntimeError(f'no port_config.ini found under {PORT_CONFIG_GLOB}') - return matches[0] - - -def build_port_to_core(port_config: Path) -> dict[str, int]: - """Parse port_config.ini and return {EthernetX: core_id}.""" - port_to_core: dict[str, int] = {} - header: list[str] | None = None - for line in port_config.read_text().splitlines(): - line = line.strip() - if not line or (line.startswith('#') and 'name' not in line): - continue - if line.startswith('#'): - header = line.lstrip('#').split() - continue - if header is None: - raise RuntimeError(f'no header in {port_config}') - fields = line.split() - if len(fields) < len(header): - continue - row = dict(zip(header, fields)) - name = row.get('name') - core_id = row.get('core_id') - if name and core_id and core_id.isdigit(): - port_to_core[name] = int(core_id) - if not port_to_core: - raise RuntimeError(f'no port->core mappings parsed from {port_config}') - return port_to_core - - -def shut_ports(ports: list[str]) -> list[tuple[str, bool, str]]: - """Shut down ports via `sudo config interface shutdown`. - - Persists admin-down state in config_db across reboots — operator must - explicitly run `sudo config interface startup EthernetX` to revert. - - Returns [(port, ok, msg)]. - """ - results = [] - for port in ports: - try: - r = subprocess.run( - SHUTDOWN_CMD + [port], - capture_output=True, text=True, timeout=15, - ) - msg = (r.stderr.strip() or r.stdout.strip()) - results.append((port, r.returncode == 0, msg)) - except subprocess.TimeoutExpired: - results.append((port, False, 'timeout')) - except Exception as e: - results.append((port, False, repr(e))) - return results - - -def open_syslog_tail() -> subprocess.Popen: - """Spawn `tail -F` on /var/log/syslog. - - `tail -F` (capital F) follows the file by name — it re-opens after log - rotation, truncation, or rsyslog restart. -n0 starts at end-of-file so - we only see new lines. - """ - return subprocess.Popen( - ['tail', '-n0', '-F', SYSLOG_PATH], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1, # line-buffered - ) - - -def parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__.splitlines()[1]) - p.add_argument('--dry-run', action='store_true', - help='log what would be shut down but do not execute') - p.add_argument('--verbose', action='store_true', - help='enable debug logging') - p.add_argument('--simulate-line', metavar='LINE', - help='for testing: inject a fake syslog line into the ' - 'detection path before tailing real syslog') - return p.parse_args() - - -def handle_match(core: int, ports_on_core: list[str], dry_run: bool, log: logging.Logger, - latched_cores: set[int]) -> None: - if core in latched_cores: - log.debug(f'core {core} already latched, ignoring duplicate event') - return - - log.critical( - f'IRE_ErrorDataPathCrc on block/core {core}; shutting ' - f'{len(ports_on_core)} ports: {ports_on_core}; manual operator ' - f'recovery required (reload syncd or reboot)' - ) - if dry_run: - log.warning(f'DRY RUN: would shut {ports_on_core}') - else: - for port, ok, msg in shut_ports(ports_on_core): - if ok: - log.info(f' shut {port}: OK') - else: - log.error(f' shut {port}: FAILED: {msg}') - latched_cores.add(core) - - -def main() -> int: - args = parse_args() - setup_logging(args.verbose) - log = logging.getLogger() - log.info(f'starting ire_watchdog (syslog-tail mode, dry_run={args.dry_run})') - - port_config = find_port_config() - log.info(f'port config: {port_config}') - port_to_core = build_port_to_core(port_config) - core_to_ports: dict[int, list[str]] = {} - for port, core in port_to_core.items(): - core_to_ports.setdefault(core, []).append(port) - for core, ports in sorted(core_to_ports.items()): - ports.sort(key=lambda p: int(p.removeprefix('Ethernet')) if p.removeprefix('Ethernet').isdigit() else 9999) - log.info(f'core {core}: {len(ports)} ports ({ports[0]}..{ports[-1]})') - - latched_cores: set[int] = set() - - # Optional injection for testing without touching real syslog - if args.simulate_line: - log.warning(f'SIMULATION: injecting line: {args.simulate_line}') - m = INTR_RE.search(args.simulate_line) - if m: - core = int(m.group(1)) - ports = core_to_ports.get(core, []) - handle_match(core, ports, args.dry_run, log, latched_cores) - else: - log.error(f'SIMULATION: line did not match regex') - - # Main loop: tail syslog, restart `tail` on EOF or error - while True: - try: - log.info(f'tailing {SYSLOG_PATH}') - proc = open_syslog_tail() - assert proc.stdout is not None - for line in proc.stdout: - m = INTR_RE.search(line) - if not m: - continue - core = int(m.group(1)) - log.info(f'matched interrupt line for core {core}: {line.strip()}') - ports = core_to_ports.get(core, []) - if not ports: - log.error(f'no ports mapped to core {core}; cannot act') - continue - handle_match(core, ports, args.dry_run, log, latched_cores) - # tail's stdout EOF — shouldn't happen with -F, but reconnect anyway - log.warning(f'tail -F exited (rc={proc.poll()}); restarting in 2s') - except Exception as e: - log.error(f'tail loop error: {e}; restarting in 5s') - time.sleep(5) - continue - time.sleep(2) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/setup.py b/setup.py index ec7c9915..df15079e 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,7 @@ 'scripts/sonic-host-server', 'scripts/ldap.py', 'scripts/console-monitor', - 'scripts/ire_watchdog', - 'scripts/ire_watchdog_syslog' + 'scripts/ire_watchdog' ], install_requires = [ 'dbus-python',