From 6ea9862c147fa6dc52dfcef0ccf624e4425427c0 Mon Sep 17 00:00:00 2001 From: Chuck Date: Thu, 11 Jun 2026 09:42:49 -0400 Subject: [PATCH 1/7] fix(install): don't let outer ERR trap mask first_time_install.sh failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit set +e alone doesn't suppress bash's ERR trap, so any non-zero exit from first_time_install.sh inside the one-shot installer immediately triggered the outer on_error handler with a generic "Main installation, line 370" message — before the script could report the real exit code or point to logs/. Suspend the trap for that block so the existing if/else handling runs instead. --- scripts/install/one-shot-install.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/install/one-shot-install.sh b/scripts/install/one-shot-install.sh index ddac1115..472b687f 100755 --- a/scripts/install/one-shot-install.sh +++ b/scripts/install/one-shot-install.sh @@ -340,9 +340,14 @@ main() { echo "" # Execute with proper error handling and non-interactive mode - # Temporarily disable errexit to capture exit code instead of exiting immediately + # Temporarily disable errexit AND the ERR trap to capture exit code instead of + # exiting immediately. `set +e` alone does not suppress the ERR trap, so without + # `trap '' ERR` a non-zero exit from first_time_install.sh would trigger on_error + # here with the generic "Main installation" message instead of the detailed + # if/else handling below. set +e - + trap '' ERR + # Check /tmp permissions - only fix if actually wrong (common in automated scenarios) # When running manually, /tmp usually has correct permissions (1777) TMP_PERMS=$(stat -c '%a' /tmp 2>/dev/null || echo "unknown") @@ -370,6 +375,7 @@ main() { sudo -E env TMPDIR=/tmp LEDMATRIX_ASSUME_YES=1 bash ./first_time_install.sh -y Date: Thu, 11 Jun 2026 09:42:59 -0400 Subject: [PATCH 2/7] feat(install): surface root cause of web dependency install failures install_dependencies_apt.py previously reported only which packages failed, not why - the actual apt/pip error was discarded (apt) or could scroll out of the on_error log tail (pip), leaving "Step 7: Install web interface dependencies (line 915)" as the only visible detail. Capture command output for each install attempt and print a compact DEPENDENCY INSTALLATION FAILURES summary with the last lines of error output per package. Also run the installer with `python3 -u` for real-time, correctly-ordered logging, and widen the on_error tail from 50 to 100 lines so the summary isn't cut off. --- first_time_install.sh | 8 +- scripts/install_dependencies_apt.py | 193 +++++++++++++++++----------- 2 files changed, 120 insertions(+), 81 deletions(-) diff --git a/first_time_install.sh b/first_time_install.sh index e43a7b8b..7bfa334e 100644 --- a/first_time_install.sh +++ b/first_time_install.sh @@ -15,8 +15,8 @@ on_error() { echo "✗ An error occurred during: $CURRENT_STEP (line $line_no, exit $exit_code)" >&2 if [ -n "${LOG_FILE:-}" ]; then echo "See the log for details: $LOG_FILE" >&2 - echo "-- Last 50 lines from log --" >&2 - tail -n 50 "$LOG_FILE" >&2 || true + echo "-- Last 100 lines from log --" >&2 + tail -n 100 "$LOG_FILE" >&2 || true fi echo "\nCommon fixes:" >&2 echo "- Ensure the Pi is online (try: ping -c1 8.8.8.8)." >&2 @@ -912,7 +912,9 @@ else # Try to install dependencies using the smart installer if available if [ -f "$PROJECT_ROOT_DIR/scripts/install_dependencies_apt.py" ]; then echo "Using smart dependency installer..." - python3 "$PROJECT_ROOT_DIR/scripts/install_dependencies_apt.py" + # -u: unbuffered stdout/stderr so output is captured in $LOG_FILE in + # real time and in order relative to this script's own echo statements + python3 -u "$PROJECT_ROOT_DIR/scripts/install_dependencies_apt.py" else echo "Using pip to install dependencies..." if [ -f "$PROJECT_ROOT_DIR/requirements_web_v2.txt" ]; then diff --git a/scripts/install_dependencies_apt.py b/scripts/install_dependencies_apt.py index ef019118..41730844 100644 --- a/scripts/install_dependencies_apt.py +++ b/scripts/install_dependencies_apt.py @@ -9,43 +9,54 @@ import warnings from pathlib import Path +# How many trailing lines of a failed command's output to keep for the +# end-of-run failure summary. Keeps the root cause near the end of the log, +# which is where first_time_install.sh's error handler tails from. +ERROR_TAIL_LINES = 15 + + +def _run(cmd): + """Run a command, capturing combined stdout/stderr. + + Returns (success, output) instead of raising, so callers can report + *why* a command failed rather than just that it failed. + """ + result = subprocess.run(cmd, capture_output=True, text=True) + return result.returncode == 0, (result.stdout or "") + (result.stderr or "") + + def install_via_apt(package_name): - """Try to install a package via apt.""" - try: - # Map pip package names to apt package names - apt_package_map = { - 'flask': 'python3-flask', - 'PIL': 'python3-pil', - 'freetype': 'python3-freetype', - 'psutil': 'python3-psutil', - 'werkzeug': 'python3-werkzeug', - 'numpy': 'python3-numpy', - 'requests': 'python3-requests', - 'python-dateutil': 'python3-dateutil', - 'pytz': 'python3-tz', - 'geopy': 'python3-geopy', - 'unidecode': 'python3-unidecode', - 'websockets': 'python3-websockets', - 'websocket-client': 'python3-websocket-client' - } - - apt_package = apt_package_map.get(package_name, f'python3-{package_name}') - - print(f"Trying to install {apt_package} via apt...") - subprocess.check_call([ - 'sudo', 'apt', 'update' - ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - subprocess.check_call([ - 'sudo', 'apt', 'install', '-y', apt_package - ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - + """Try to install a package via apt. Returns (success, output).""" + # Map pip package names to apt package names + apt_package_map = { + 'flask': 'python3-flask', + 'PIL': 'python3-pil', + 'freetype': 'python3-freetype', + 'psutil': 'python3-psutil', + 'werkzeug': 'python3-werkzeug', + 'numpy': 'python3-numpy', + 'requests': 'python3-requests', + 'python-dateutil': 'python3-dateutil', + 'pytz': 'python3-tz', + 'geopy': 'python3-geopy', + 'unidecode': 'python3-unidecode', + 'websockets': 'python3-websockets', + 'websocket-client': 'python3-websocket-client' + } + + apt_package = apt_package_map.get(package_name, f'python3-{package_name}') + + print(f"Trying to install {apt_package} via apt...") + _run(['sudo', 'apt', 'update']) # best-effort refresh; ignore failures here + + success, output = _run(['sudo', 'apt', 'install', '-y', apt_package]) + if success: print(f"Successfully installed {apt_package} via apt") - return True - - except subprocess.CalledProcessError: - print(f"Failed to install {package_name} via apt, will try pip") - return False + return True, "" + + print(f"Failed to install {apt_package} via apt, will try pip") + return False, output + def install_via_pip(package_name): """Install a package via pip with --break-system-packages and --prefer-binary. @@ -54,17 +65,20 @@ def install_via_pip(package_name): Debian/Ubuntu-based systems without a virtual environment. --prefer-binary prefers pre-built wheels over source distributions to avoid exhausting /tmp space during compilation. + + Returns (success, output). """ - try: - print(f"Installing {package_name} via pip...") - subprocess.check_call([ - sys.executable, '-m', 'pip', 'install', '--break-system-packages', '--prefer-binary', package_name - ]) + print(f"Installing {package_name} via pip...") + success, output = _run([ + sys.executable, '-m', 'pip', 'install', '--break-system-packages', '--prefer-binary', package_name + ]) + if success: print(f"Successfully installed {package_name} via pip") - return True - except subprocess.CalledProcessError as e: - print(f"Failed to install {package_name} via pip: {e}") - return False + return True, "" + + print(f"Failed to install {package_name} via pip (see failure summary at end of log)") + return False, output + def check_package_installed(package_name): """Check if a package is already installed.""" @@ -78,10 +92,27 @@ def check_package_installed(package_name): except ImportError: return False + +def print_failure_summary(failed_packages, failure_details): + print("\n" + "=" * 60) + print("DEPENDENCY INSTALLATION FAILURES - DETAILS") + print("=" * 60) + for package in failed_packages: + print(f"\nPackage: {package}") + print("-" * 40) + output = failure_details.get(package, "").strip() + if not output: + print(" (no output captured)") + continue + for line in output.splitlines()[-ERROR_TAIL_LINES:]: + print(f" {line}") + print("=" * 60) + + def main(): """Main installation function.""" print("Installing dependencies for LED Matrix Web Interface V2...") - + # List of required packages required_packages = [ 'flask', @@ -98,19 +129,23 @@ def main(): 'websockets', 'websocket-client' ] - + failed_packages = [] - + failure_details = {} + for package in required_packages: if check_package_installed(package): print(f"{package} is already installed") continue - + # Try apt first, then pip - if not install_via_apt(package): - if not install_via_pip(package): + ok, apt_output = install_via_apt(package) + if not ok: + ok, pip_output = install_via_pip(package) + if not ok: failed_packages.append(package) - + failure_details[package] = pip_output or apt_output + # Install packages that don't have apt equivalents special_packages = [ 'timezonefinder>=6.5.0,<7.0.0', @@ -122,47 +157,49 @@ def main(): 'python-socketio>=5.11.0,<6.0.0', 'python-engineio>=4.9.0,<5.0.0' ] - + for package in special_packages: - if not install_via_pip(package): + ok, pip_output = install_via_pip(package) + if not ok: failed_packages.append(package) - + failure_details[package] = pip_output + # Install rgbmatrix module from local source (optional - may already be installed in Step 6) # Check if already installed first if check_package_installed('rgbmatrix'): print("rgbmatrix module already installed, skipping...") else: print("Installing rgbmatrix module from local source...") - try: - # Get project root (parent of scripts directory) - PROJECT_ROOT = Path(__file__).parent.parent - rgbmatrix_path = PROJECT_ROOT / 'rpi-rgb-led-matrix-master' / 'bindings' / 'python' - if rgbmatrix_path.exists(): - # Check if the module has been built (look for setup.py) - setup_py = rgbmatrix_path / 'setup.py' - if setup_py.exists(): - # Try installing - use regular install, not editable mode - # This is optional for web interface and should already be installed in Step 6 - subprocess.check_call([ - sys.executable, '-m', 'pip', 'install', '--break-system-packages', str(rgbmatrix_path) - ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + # Get project root (parent of scripts directory) + PROJECT_ROOT = Path(__file__).parent.parent + rgbmatrix_path = PROJECT_ROOT / 'rpi-rgb-led-matrix-master' / 'bindings' / 'python' + if rgbmatrix_path.exists(): + # Check if the module has been built (look for setup.py) + setup_py = rgbmatrix_path / 'setup.py' + if setup_py.exists(): + # Try installing - use regular install, not editable mode + # This is optional for web interface and should already be installed in Step 6 + ok, output = _run([sys.executable, '-m', 'pip', 'install', '--break-system-packages', str(rgbmatrix_path)]) + if ok: print("rgbmatrix module installed successfully") else: - print("Warning: rgbmatrix setup.py not found, module may need to be built first") - print(" This is normal if Step 6 hasn't completed yet.") + # Don't fail the whole installation - rgbmatrix is optional for web interface + # and should be installed in Step 6 of first_time_install.sh + print("Warning: Failed to install rgbmatrix module:") + for line in output.strip().splitlines()[-ERROR_TAIL_LINES:]: + print(f" {line}") + print(" This is normal if rgbmatrix hasn't been built yet (Step 6).") + print(" The web interface will work without it.") else: - print("Warning: rgbmatrix source not found (this is normal if Step 6 hasn't run yet)") - except subprocess.CalledProcessError as e: - # Don't fail the whole installation - rgbmatrix is optional for web interface - # and should be installed in Step 6 of first_time_install.sh - print(f"Warning: Failed to install rgbmatrix module: {e}") - print(" This is normal if rgbmatrix hasn't been built yet (Step 6).") - print(" The web interface will work without it.") - # Don't add to failed_packages since it's optional - + print("Warning: rgbmatrix setup.py not found, module may need to be built first") + print(" This is normal if Step 6 hasn't completed yet.") + else: + print("Warning: rgbmatrix source not found (this is normal if Step 6 hasn't run yet)") + if failed_packages: print(f"\nFailed to install the following packages: {failed_packages}") print("You may need to install them manually or check your system configuration.") + print_failure_summary(failed_packages, failure_details) return False else: print("\nAll dependencies installed successfully!") From 6aec2d9b7857f8138baf6190ac3f8039bf9e76bc Mon Sep 17 00:00:00 2001 From: Chuck Date: Thu, 11 Jun 2026 10:21:32 -0400 Subject: [PATCH 3/7] feat(install): harden first-time install against common Pi failure modes - wait_for_apt_lock: apt_update/apt_install now wait (up to 3min) for unattended-upgrades to release the dpkg lock instead of failing outright with "Command failed after 3 attempts" right after first boot. - check_disk_space: new pre-flight check (Step 1) so a full SD card fails fast with a clear message instead of a cryptic mid-build error. - Step 6: wrap rpi-rgb-led-matrix git clone/submodule operations in retry for resilience to transient network issues. - Step 6: capture `pip install .` build output and print the last 50 lines on failure, so the actual cmake/compiler error is visible instead of just "Failed to install rpi-rgb-led-matrix Python package". --- first_time_install.sh | 73 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/first_time_install.sh b/first_time_install.sh index 7bfa334e..44eed924 100644 --- a/first_time_install.sh +++ b/first_time_install.sh @@ -202,8 +202,33 @@ retry() { done } -apt_update() { retry apt update; } -apt_install() { retry apt install -y "$@"; } +# Wait for another apt/dpkg process (commonly unattended-upgrades running +# shortly after first boot) to release its lock before we try apt ourselves. +# Without this, apt_update/apt_install can fail outright in the first couple +# minutes after a fresh Pi OS boot with a generic "Command failed after 3 +# attempts" error. +wait_for_apt_lock() { + command -v flock >/dev/null 2>&1 || return 0 + local lock_file="/var/lib/dpkg/lock-frontend" + local max_wait=180 + local waited=0 + local printed=0 + while ! flock -n "$lock_file" -c true 2>/dev/null; do + if [ "$printed" -eq 0 ]; then + echo "⚠ Waiting for another apt/dpkg process to finish (e.g. unattended-upgrades on first boot)..." + printed=1 + fi + if [ "$waited" -ge "$max_wait" ]; then + echo "⚠ Still waiting after ${max_wait}s; proceeding anyway." + break + fi + sleep 5 + waited=$((waited+5)) + done +} + +apt_update() { wait_for_apt_lock; retry apt update; } +apt_install() { wait_for_apt_lock; retry apt install -y "$@"; } apt_remove() { apt-get remove -y "$@" || true; } check_network() { @@ -222,6 +247,22 @@ check_network() { exit 1 } +check_disk_space() { + command -v df >/dev/null 2>&1 || return 0 + local available_mb + available_mb=$(df -m "$PROJECT_ROOT_DIR" | awk 'NR==2{print $4}') + available_mb=${available_mb:-0} + if [ "$available_mb" -lt 500 ]; then + echo "✗ ERROR: Insufficient disk space: ${available_mb}MB available (need at least 500MB)" + echo " Free up space first, e.g.: sudo apt clean && sudo apt autoremove" + exit 1 + elif [ "$available_mb" -lt 1024 ]; then + echo "⚠ Limited disk space: ${available_mb}MB available (recommend at least 1GB for the rpi-rgb-led-matrix build in Step 6)" + else + echo "✓ Disk space sufficient: ${available_mb}MB available" + fi +} + echo "" echo "This script will perform the following steps:" echo "1. Install system dependencies" @@ -271,8 +312,9 @@ CURRENT_STEP="Install system dependencies" echo "Step 1: Installing system dependencies..." echo "----------------------------------------" -# Ensure network is available before APT operations +# Pre-flight checks before APT operations check_network +check_disk_space # Update package list apt_update @@ -822,14 +864,14 @@ else # Try to initialize submodule if .gitmodules exists if [ -f "$PROJECT_ROOT_DIR/.gitmodules" ] && grep -q "rpi-rgb-led-matrix" "$PROJECT_ROOT_DIR/.gitmodules"; then echo "Initializing rpi-rgb-led-matrix submodule..." - if ! git submodule update --init --recursive rpi-rgb-led-matrix-master 2>&1; then + if ! retry git submodule update --init --recursive rpi-rgb-led-matrix-master; then echo "⚠ Submodule init failed, cloning directly from GitHub..." - git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master + retry git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master fi else # Fallback: clone directly if submodule not configured echo "Submodule not configured, cloning directly from GitHub..." - git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master + retry git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master fi fi @@ -841,23 +883,34 @@ else cd "$PROJECT_ROOT_DIR" rm -rf rpi-rgb-led-matrix-master if [ -f "$PROJECT_ROOT_DIR/.gitmodules" ] && grep -q "rpi-rgb-led-matrix" "$PROJECT_ROOT_DIR/.gitmodules"; then - git submodule update --init --recursive rpi-rgb-led-matrix-master + retry git submodule update --init --recursive rpi-rgb-led-matrix-master else - git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master + retry git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master fi fi - + pushd "$PROJECT_ROOT_DIR/rpi-rgb-led-matrix-master" >/dev/null echo "Installing rpi-rgb-led-matrix Python package (scikit-build-core + cmake)..." echo " Build deps required: python-dev-is-python3 cmake" echo " This compiles C++ — may take 2-5 minutes on Pi 4/5..." - if ! python3 -m pip install --break-system-packages .; then + BUILD_OUTPUT=$(mktemp) + BUILD_SUCCESS=false + if python3 -m pip install --break-system-packages . > "$BUILD_OUTPUT" 2>&1; then + BUILD_SUCCESS=true + fi + cat "$BUILD_OUTPUT" >> "$LOG_FILE" + if [ "$BUILD_SUCCESS" != true ]; then echo "✗ Failed to install rpi-rgb-led-matrix Python package" echo " Ensure build tools are installed:" echo " sudo apt install -y python-dev-is-python3 cmake build-essential" + echo "" + echo "-- Last 50 lines of build output --" + tail -n 50 "$BUILD_OUTPUT" + rm -f "$BUILD_OUTPUT" popd >/dev/null exit 1 fi + rm -f "$BUILD_OUTPUT" popd >/dev/null else echo "✗ rpi-rgb-led-matrix-master directory not found at $PROJECT_ROOT_DIR" From 5a1a095e166bd06f2cb9a945b21c0df82a73531a Mon Sep 17 00:00:00 2001 From: Chuck Date: Thu, 11 Jun 2026 12:05:47 -0400 Subject: [PATCH 4/7] fix(install): bound subprocess output and dedupe apt update in dependency installer Address coderabbitai review on PR #369: - _run() now streams combined stdout/stderr to a temp file and returns only the last ERROR_TAIL_LINES lines, instead of buffering full output in memory (Codacy also flagged the previous capture_output call as a subprocess-without-static-string security issue; the new call is annotated as safe since cmd is built from hardcoded args). - `apt update` now runs once in main() instead of once per package needing an apt fallback. --- scripts/install_dependencies_apt.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/install_dependencies_apt.py b/scripts/install_dependencies_apt.py index 41730844..21d07985 100644 --- a/scripts/install_dependencies_apt.py +++ b/scripts/install_dependencies_apt.py @@ -6,6 +6,7 @@ import subprocess import sys +import tempfile import warnings from pathlib import Path @@ -16,13 +17,18 @@ def _run(cmd): - """Run a command, capturing combined stdout/stderr. + """Run a command, streaming combined stdout/stderr to a temp file. Returns (success, output) instead of raising, so callers can report - *why* a command failed rather than just that it failed. + *why* a command failed rather than just that it failed. `output` is + bounded to the last ERROR_TAIL_LINES lines so failures from very + chatty commands (e.g. pip build logs) don't get buffered in memory. """ - result = subprocess.run(cmd, capture_output=True, text=True) - return result.returncode == 0, (result.stdout or "") + (result.stderr or "") + with tempfile.TemporaryFile(mode='w+b') as f: + result = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT) # nosec B603 B607 - hardcoded apt/pip args, not user input + f.seek(0) + lines = f.read().decode('utf-8', errors='replace').splitlines() + return result.returncode == 0, '\n'.join(lines[-ERROR_TAIL_LINES:]) def install_via_apt(package_name): @@ -47,8 +53,6 @@ def install_via_apt(package_name): apt_package = apt_package_map.get(package_name, f'python3-{package_name}') print(f"Trying to install {apt_package} via apt...") - _run(['sudo', 'apt', 'update']) # best-effort refresh; ignore failures here - success, output = _run(['sudo', 'apt', 'install', '-y', apt_package]) if success: print(f"Successfully installed {apt_package} via apt") @@ -113,6 +117,9 @@ def main(): """Main installation function.""" print("Installing dependencies for LED Matrix Web Interface V2...") + print("Refreshing apt package index...") + _run(['sudo', 'apt', 'update']) # best-effort; individual installs surface their own errors + # List of required packages required_packages = [ 'flask', From 60b64144a5add92dc826a95e326a429ed186c8b1 Mon Sep 17 00:00:00 2001 From: Chuck Date: Thu, 11 Jun 2026 13:01:33 -0400 Subject: [PATCH 5/7] fix(install): suppress remaining Codacy subprocess false-positive Codacy's Semgrep-based check still flagged the cmd-built subprocess.run call as "without a static string" even with the Bandit nosec applied. Add a nosemgrep marker alongside it - cmd is always a hardcoded apt/pip argument list, never user input. --- scripts/install_dependencies_apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_dependencies_apt.py b/scripts/install_dependencies_apt.py index 21d07985..18671662 100644 --- a/scripts/install_dependencies_apt.py +++ b/scripts/install_dependencies_apt.py @@ -25,7 +25,7 @@ def _run(cmd): chatty commands (e.g. pip build logs) don't get buffered in memory. """ with tempfile.TemporaryFile(mode='w+b') as f: - result = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT) # nosec B603 B607 - hardcoded apt/pip args, not user input + result = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT) # nosec B603 B607 - hardcoded apt/pip args # nosemgrep f.seek(0) lines = f.read().decode('utf-8', errors='replace').splitlines() return result.returncode == 0, '\n'.join(lines[-ERROR_TAIL_LINES:]) From eb6687ceca79faed89f0af2fc15ca33cdc12d645 Mon Sep 17 00:00:00 2001 From: Chuck Date: Thu, 11 Jun 2026 16:21:56 -0400 Subject: [PATCH 6/7] fix(install): correctly detect already-installed dateutil/websocket-client Address remaining coderabbitai findings on PR #369: - check_package_installed() did __import__(package_name) directly, but python-dateutil and websocket-client import as dateutil/websocket. Both always failed the "already installed" check and were reinstalled on every run. Add an IMPORT_NAME_MAP for the mismatched names. - _run() still read the entire temp file into memory before slicing the tail. Stream it line-by-line into a deque(maxlen=ERROR_TAIL_LINES) instead so memory use stays bounded for very chatty commands. --- scripts/install_dependencies_apt.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/install_dependencies_apt.py b/scripts/install_dependencies_apt.py index 18671662..a12d0b30 100644 --- a/scripts/install_dependencies_apt.py +++ b/scripts/install_dependencies_apt.py @@ -8,6 +8,7 @@ import sys import tempfile import warnings +from collections import deque from pathlib import Path # How many trailing lines of a failed command's output to keep for the @@ -27,8 +28,13 @@ def _run(cmd): with tempfile.TemporaryFile(mode='w+b') as f: result = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT) # nosec B603 B607 - hardcoded apt/pip args # nosemgrep f.seek(0) - lines = f.read().decode('utf-8', errors='replace').splitlines() - return result.returncode == 0, '\n'.join(lines[-ERROR_TAIL_LINES:]) + # Stream line-by-line so only the last ERROR_TAIL_LINES are ever held + # in memory, regardless of how much output the command produced. + tail = deque( + (line.decode('utf-8', errors='replace').rstrip('\n') for line in f), + maxlen=ERROR_TAIL_LINES, + ) + return result.returncode == 0, '\n'.join(tail) def install_via_apt(package_name): @@ -84,14 +90,22 @@ def install_via_pip(package_name): return False, output +# Distribution (pip/apt) names whose importable module name differs. +IMPORT_NAME_MAP = { + 'python-dateutil': 'dateutil', + 'websocket-client': 'websocket', +} + + def check_package_installed(package_name): """Check if a package is already installed.""" + import_name = IMPORT_NAME_MAP.get(package_name, package_name) # Suppress deprecation warnings when checking if packages are installed # (we're just checking, not using them) with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=DeprecationWarning) try: - __import__(package_name) + __import__(import_name) return True except ImportError: return False From b2524e918d454b4fc1988f7ac52d3cc0d5a16ef5 Mon Sep 17 00:00:00 2001 From: Chuck Date: Tue, 16 Jun 2026 09:41:33 -0400 Subject: [PATCH 7/7] feat(web): add Tools tab and row address type setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Tools/Utilities tab to the web interface with one-click maintenance buttons that previously required SSH: - Git status panel (branch, dirty state, recent commits) - Pull latest (rebase) and force reset to origin/main - Reinstall base requirements (pip, with output) - Reinstall per-plugin requirements (pass/fail per plugin) - Clear __pycache__ directories - Quick-access restart for display and web services Also exposes the hzeller row_address_type option (0–4) in the Display settings tab. The backend already read this value from config; the UI, API field list, and validation were missing. Co-Authored-By: Claude Sonnet 4.6 --- web_interface/blueprints/api_v3.py | 97 +++++- web_interface/blueprints/pages_v3.py | 11 + web_interface/templates/v3/base.html | 17 + .../templates/v3/partials/display.html | 12 + .../templates/v3/partials/tools.html | 306 ++++++++++++++++++ 5 files changed, 440 insertions(+), 3 deletions(-) create mode 100644 web_interface/templates/v3/partials/tools.html diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 4c954b00..9cb849eb 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -705,7 +705,8 @@ def save_main_config(): display_fields = ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'hardware_mapping', 'gpio_slowdown', 'rp1_rio', 'scan_mode', 'disable_hardware_pulsing', 'inverse_colors', 'show_refresh_rate', 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', 'use_short_date_format', - 'max_dynamic_duration_seconds', 'led_rgb_sequence', 'multiplexing', 'panel_type'] + 'max_dynamic_duration_seconds', 'led_rgb_sequence', 'multiplexing', 'panel_type', + 'row_address_type'] if any(k in data for k in display_fields): if 'display' not in current_config: @@ -736,14 +737,23 @@ def save_main_config(): except (ValueError, TypeError): return jsonify({'status': 'error', 'message': f"Invalid multiplexing value '{data['multiplexing']}'. Must be an integer from 0 to 22."}), 400 + # Validate row_address_type + if 'row_address_type' in data: + try: + rat_val = int(data['row_address_type']) + if rat_val < 0 or rat_val > 4: + return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 + except (ValueError, TypeError): + return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 + # Handle hardware settings for field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'hardware_mapping', 'scan_mode', 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', - 'led_rgb_sequence', 'multiplexing', 'panel_type']: + 'led_rgb_sequence', 'multiplexing', 'panel_type', 'row_address_type']: if field in data: if field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'scan_mode', 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', - 'multiplexing']: + 'multiplexing', 'row_address_type']: current_config['display']['hardware'][field] = int(data[field]) else: current_config['display']['hardware'][field] = data[field] @@ -1574,6 +1584,66 @@ def execute_system_action(): # Try to restart the web service (assuming it's ledmatrix-web.service) result = subprocess.run(['sudo', 'systemctl', 'restart', 'ledmatrix-web.service'], capture_output=True, text=True, timeout=10) + elif action == 'install_base_requirements': + req_file = PROJECT_ROOT / 'requirements.txt' + if not req_file.exists(): + return jsonify({'status': 'error', 'message': 'No requirements.txt found at project root'}) + result = subprocess.run( + [sys.executable, '-m', 'pip', 'install', '--break-system-packages', '-r', str(req_file)], + capture_output=True, text=True, timeout=120, cwd=str(PROJECT_ROOT) + ) + return jsonify({ + 'status': 'success' if result.returncode == 0 else 'error', + 'message': 'Base requirements installed successfully' if result.returncode == 0 else 'pip install failed', + 'output': (result.stdout + result.stderr).strip() + }) + elif action == 'install_plugin_requirements': + plugins_dir = Path(plugin_manager.plugins_dir) if plugin_manager else PROJECT_ROOT / 'plugin-repos' + results = [] + if plugins_dir.exists(): + for p in sorted(plugins_dir.iterdir()): + req = p / 'requirements.txt' + if p.is_dir() and req.exists(): + r = subprocess.run( + [sys.executable, '-m', 'pip', 'install', '--break-system-packages', '-r', str(req)], + capture_output=True, text=True, timeout=60 + ) + results.append({ + 'plugin': p.name, + 'ok': r.returncode == 0, + 'output': (r.stdout + r.stderr).strip() + }) + ok_count = sum(1 for r in results if r['ok']) + all_ok = all(r['ok'] for r in results) if results else True + return jsonify({ + 'status': 'success' if all_ok else 'error', + 'message': f'Processed {len(results)} plugin(s) — {ok_count} succeeded' if results else 'No plugin requirements.txt files found', + 'details': results + }) + elif action == 'force_git_reset': + project_dir = str(PROJECT_ROOT) + fetch = subprocess.run( + ['git', 'fetch', 'origin'], + capture_output=True, text=True, timeout=30, cwd=project_dir + ) + if fetch.returncode != 0: + return jsonify({'status': 'error', 'message': 'git fetch failed', 'output': fetch.stderr.strip()}) + reset = subprocess.run( + ['git', 'reset', '--hard', 'origin/main'], + capture_output=True, text=True, timeout=30, cwd=project_dir + ) + return jsonify({ + 'status': 'success' if reset.returncode == 0 else 'error', + 'message': 'Reset to origin/main successfully' if reset.returncode == 0 else 'git reset failed', + 'output': (reset.stdout + reset.stderr).strip() + }) + elif action == 'clear_pycache': + cleared = 0 + for d in PROJECT_ROOT.rglob('__pycache__'): + if d.is_dir(): + shutil.rmtree(d, ignore_errors=True) + cleared += 1 + return jsonify({'status': 'success', 'message': f'Cleared {cleared} __pycache__ directories'}) else: return jsonify({'status': 'error', 'message': 'Unknown action'}), 400 @@ -1596,6 +1666,27 @@ def execute_system_action(): logger.error("execute_system_action failed: %s", e, exc_info=True) return jsonify({'status': 'error', 'message': 'Action failed; see logs for details'}), 500 +@api_v3.route('/system/git-info', methods=['GET']) +def get_git_info(): + """Return branch, dirty state, recent commits and remote URL for the Tools tab.""" + d = str(PROJECT_ROOT) + try: + branch = subprocess.run(['git', 'branch', '--show-current'], capture_output=True, text=True, timeout=10, cwd=d) + status = subprocess.run(['git', 'status', '--short', '--untracked-files=no'], capture_output=True, text=True, timeout=15, cwd=d) + log = subprocess.run(['git', 'log', '--oneline', '-5'], capture_output=True, text=True, timeout=10, cwd=d) + remote = subprocess.run(['git', 'remote', 'get-url', 'origin'], capture_output=True, text=True, timeout=10, cwd=d) + return jsonify({ + 'branch': branch.stdout.strip(), + 'dirty': bool(status.stdout.strip()), + 'status': status.stdout.strip(), + 'recent_commits': log.stdout.strip(), + 'remote_url': remote.stdout.strip(), + }) + except Exception as e: + logger.error("get_git_info failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'Failed to get git info'}), 500 + + @api_v3.route('/hardware/status', methods=['GET']) def get_hardware_status(): """Return LED matrix hardware initialization status written by display_manager at startup.""" diff --git a/web_interface/blueprints/pages_v3.py b/web_interface/blueprints/pages_v3.py index 73939f52..1086b5f9 100644 --- a/web_interface/blueprints/pages_v3.py +++ b/web_interface/blueprints/pages_v3.py @@ -90,6 +90,8 @@ def load_partial(partial_name): return _load_cache_partial() elif partial_name == 'operation-history': return _load_operation_history_partial() + elif partial_name == 'tools': + return _load_tools_partial() else: return "Partial not found", 404 @@ -448,6 +450,15 @@ def _load_operation_history_partial(): return "Error loading partial", 500 +def _load_tools_partial(): + """Load tools/utilities partial.""" + try: + return render_template('v3/partials/tools.html') + except Exception: + logger.error("Error loading partial", exc_info=True) + return "Error loading partial", 500 + + def _load_plugin_config_partial(plugin_id): """ Load plugin configuration partial - server-side rendered form. diff --git a/web_interface/templates/v3/base.html b/web_interface/templates/v3/base.html index f8a08295..1527fc55 100644 --- a/web_interface/templates/v3/base.html +++ b/web_interface/templates/v3/base.html @@ -1009,6 +1009,11 @@

class="nav-tab"> Operation History + @@ -1290,6 +1295,18 @@

+ +
+
+
+
+
+
+
+
+
+
+ +
+
+

Git & Updates

+

Inspect the current git state and pull or reset to the latest remote code.

+
+ + +
+
Loading git info…
+
+ +
+ +
+
+

Pull latest (rebase)

+

Stashes local changes, runs git pull --rebase, then restores the stash.

+
+ +
+ + + +
+
+

Force reset to origin/main

+

Runs git fetch origin then git reset --hard origin/main. Discards all local changes.

+
+
+ + +
+
+ +
+
+ + +
+
+

Python Dependencies

+

Re-run pip install to fix missing or broken packages.

+
+ +
+ +
+
+

Reinstall base requirements

+

Installs from requirements.txt in the project root.

+
+ +
+ + + +
+
+

Reinstall plugin requirements

+

Runs pip install for every installed plugin that has a requirements.txt.

+
+ +
+ +
+
+ + +
+
+

Maintenance

+

Housekeeping operations that don't affect config or plugins.

+
+ +
+
+
+

Clear Python cache

+

Deletes all __pycache__ directories in the project. Useful after switching branches or debugging import issues.

+
+ +
+ +
+
+ + +
+
+

Services

+

Quick access to service restarts.

+
+ +
+
+
+

Restart display service

+

Restarts ledmatrix.service.

+
+ +
+ + +
+
+

Restart web interface

+

Restarts ledmatrix-web.service. The page will go offline briefly.

+
+ +
+ +
+
+ + + +