From 6707590923ad69bb802bd4229e0263e747bac7df Mon Sep 17 00:00:00 2001 From: Journey Date: Mon, 24 Nov 2025 14:18:32 -0600 Subject: [PATCH 1/4] Convert to python --- .dockerignore | 42 ++++ .github/workflows/docker-publish-dev.yml | 19 ++ .gitignore | 48 ++++ Dockerfile | 22 +- autoheal/__init__.py | 3 + autoheal/__main__.py | 55 +++++ autoheal/config.py | 56 +++++ autoheal/docker_api.py | 60 +++++ autoheal/health_monitor.py | 61 +++++ autoheal/logging_utils.py | 45 ++++ autoheal/notifications.py | 47 ++++ autoheal/restart_tracker.py | 57 +++++ docker-entrypoint | 270 ----------------------- pyproject.toml | 16 ++ tests/.env | 1 - tests/README.md | 24 -- tests/docker-compose.autoheal.yml | 12 - tests/docker-compose.yml | 57 ----- tests/tests.sh | 19 -- tests/watch-autoheal/Dockerfile | 9 - tests/watch-autoheal/entrypoint.sh | 20 -- 21 files changed, 523 insertions(+), 420 deletions(-) create mode 100644 .github/workflows/docker-publish-dev.yml create mode 100644 autoheal/__init__.py create mode 100644 autoheal/__main__.py create mode 100644 autoheal/config.py create mode 100644 autoheal/docker_api.py create mode 100644 autoheal/health_monitor.py create mode 100644 autoheal/logging_utils.py create mode 100644 autoheal/notifications.py create mode 100644 autoheal/restart_tracker.py delete mode 100755 docker-entrypoint create mode 100644 pyproject.toml delete mode 100755 tests/.env delete mode 100755 tests/README.md delete mode 100755 tests/docker-compose.autoheal.yml delete mode 100755 tests/docker-compose.yml delete mode 100755 tests/tests.sh delete mode 100755 tests/watch-autoheal/Dockerfile delete mode 100755 tests/watch-autoheal/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index b1e4a7c..7cb0d38 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,35 @@ .vscode/ tests/ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +.venv/ +venv/ +ENV/ +env/ +.coverage +.pytest_cache/ +htmlcov/ + # Ignore common archive and executable files *.exe *.zip @@ -20,7 +49,20 @@ tests/ README.md LICENSE renovate.json +pyproject.toml # Docker-specific files Dockerfile docker-compose.yml +.dockerignore + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log +autoheal_logs.txt + +# Environment +.env diff --git a/.github/workflows/docker-publish-dev.yml b/.github/workflows/docker-publish-dev.yml new file mode 100644 index 0000000..620b839 --- /dev/null +++ b/.github/workflows/docker-publish-dev.yml @@ -0,0 +1,19 @@ +name: Docker Publish Dev + +on: + workflow_dispatch: + push: + branches: + - dev + +permissions: + packages: write + +jobs: + call-reusable-workflow: + uses: JourneyDocker/github-workflows/.github/workflows/docker-publish.yml@main + with: + tag: dev + platforms: linux/amd64 + registry: github + secrets: inherit diff --git a/.gitignore b/.gitignore index 6560709..110af20 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,51 @@ # IDE .idea *.iml + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# Testing +.coverage +.pytest_cache/ +htmlcov/ + +# Docker +.dockerignore + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log +autoheal_logs.txt + +# Environment +.env diff --git a/Dockerfile b/Dockerfile index fd9a7d4..c96d80b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,16 @@ -FROM alpine:3.22.2 +FROM python:3.14.0-alpine # Install required packages -RUN apk add --no-cache curl jq tzdata +RUN apk add --no-cache curl tzdata procps + +# Set working directory +WORKDIR /app + +# Copy source code +COPY autoheal/ /app/autoheal/ + +# Install Python dependencies +RUN pip install --no-cache-dir requests docker # Environment variables ENV AUTOHEAL_CONTAINER_LABEL=autoheal \ @@ -13,16 +22,13 @@ ENV AUTOHEAL_CONTAINER_LABEL=autoheal \ DOCKER_SOCK=/var/run/docker.sock \ CURL_TIMEOUT=30 \ WEBHOOK_URL="" \ - WEBHOOK_JSON_KEY="text" \ + WEBHOOK_JSON_ENTRY="text" \ APPRISE_URL="" \ POST_RESTART_SCRIPT="" -# Copy entrypoint script -COPY docker-entrypoint / - # Health check to ensure the process is running -HEALTHCHECK --interval=5s CMD pgrep -f autoheal || exit 1 +HEALTHCHECK --interval=5s CMD pgrep -f python || exit 1 # Set entrypoint and default command -ENTRYPOINT ["/docker-entrypoint"] +ENTRYPOINT ["python", "-m", "autoheal"] CMD ["autoheal"] diff --git a/autoheal/__init__.py b/autoheal/__init__.py new file mode 100644 index 0000000..630bf36 --- /dev/null +++ b/autoheal/__init__.py @@ -0,0 +1,3 @@ +"""Docker Autoheal - Monitor and automatically restart unhealthy Docker containers.""" + +__version__ = "2.0.0" \ No newline at end of file diff --git a/autoheal/__main__.py b/autoheal/__main__.py new file mode 100644 index 0000000..c4e9abb --- /dev/null +++ b/autoheal/__main__.py @@ -0,0 +1,55 @@ +"""Main entry point for Docker Autoheal.""" + +import signal +import sys +import time +import os +from .config import Config +from .health_monitor import monitor_containers +from .logging_utils import setup_logging + + +def signal_handler(signum, frame): + """Handle SIGTERM.""" + logger.info("Received SIGTERM signal, shutting down") + sys.exit(143) + + +def main(): + """Main function.""" + config = Config() + logger = setup_logging(config.log_level) + + logger.info(f"Docker Autoheal v{__import__('autoheal').__version__} starting up") + logger.info("Configuration:") + for key, value in config.__dict__.items(): + if key.lower() == 'webhook_url': + # Mask sensitive URLs + str_value = str(value) + if str_value: + masked_value = str_value[:20] + '...' if len(str_value) > 20 else '***' + else: + masked_value = str_value + else: + masked_value = str(value) + logger.info(f" {key.upper()}={masked_value}") + + if config.unix_sock and not os.path.exists(config.docker_sock): + logger.error("Unix socket is currently not available") + sys.exit(1) + + signal.signal(signal.SIGTERM, signal_handler) + + if config.autoheal_start_period > 0: + logger.info(f"Waiting {config.autoheal_start_period} second(s) before starting monitoring") + time.sleep(config.autoheal_start_period) + + monitor_containers(config) + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "autoheal": + main() + else: + # Exec other commands + os.execvp(sys.argv[1], sys.argv[1:]) \ No newline at end of file diff --git a/autoheal/config.py b/autoheal/config.py new file mode 100644 index 0000000..ad2a063 --- /dev/null +++ b/autoheal/config.py @@ -0,0 +1,56 @@ +"""Configuration management for Docker Autoheal.""" + +import os +from dataclasses import dataclass, field + + +@dataclass +class Config: + """Configuration class for environment variables.""" + + docker_sock: str = field(default_factory=lambda: os.getenv("DOCKER_SOCK", "/var/run/docker.sock")) + curl_timeout: int = field(default_factory=lambda: int(os.getenv("CURL_TIMEOUT", "30"))) + webhook_url: str = field(default_factory=lambda: os.getenv("WEBHOOK_URL", "")) + webhook_json_entry: str = field(default_factory=lambda: os.getenv("WEBHOOK_JSON_ENTRY", "text")) + apprise_url: str = field(default_factory=lambda: os.getenv("APPRISE_URL", "")) + post_restart_script: str = field(default_factory=lambda: os.getenv("POST_RESTART_SCRIPT", "")) + autoheal_container_label: str = field(default_factory=lambda: os.getenv("AUTOHEAL_CONTAINER_LABEL", "autoheal")) + autoheal_start_period: int = field(default_factory=lambda: int(os.getenv("AUTOHEAL_START_PERIOD", "0"))) + autoheal_interval: int = field(default_factory=lambda: int(os.getenv("AUTOHEAL_INTERVAL", "5"))) + autoheal_default_stop_timeout: int = field(default_factory=lambda: int(os.getenv("AUTOHEAL_DEFAULT_STOP_TIMEOUT", "10"))) + autoheal_only_monitor_running: bool = field(default_factory=lambda: os.getenv("AUTOHEAL_ONLY_MONITOR_RUNNING", "false").lower() == "true") + autoheal_restart_threshold: int = field(default_factory=lambda: int(os.getenv("AUTOHEAL_RESTART_THRESHOLD", "5"))) + autoheal_restart_window: int = field(default_factory=lambda: int(os.getenv("AUTOHEAL_RESTART_WINDOW", "600"))) + log_level: int = field(default_factory=lambda: int(os.getenv("LOG_LEVEL", "1"))) # 1=INFO, 2=DEBUG + + @property + def http_endpoint(self) -> str: + """Determine the HTTP endpoint based on DOCKER_SOCK.""" + if self.docker_sock.startswith("tcp://"): + return self.docker_sock.replace("tcp://", "http://", 1) + elif self.docker_sock.startswith("tcps://"): + return self.docker_sock.replace("tcps://", "https://", 1) + else: + return "http://localhost" + + @property + def unix_sock(self) -> str: + """Return unix socket path if applicable.""" + if not self.docker_sock.startswith(("tcp://", "tcps://")): + return self.docker_sock + return "" + + @property + def ca_cert(self) -> str: + """CA cert path for TLS.""" + return "/certs/ca.pem" if self.docker_sock.startswith("tcps://") else "" + + @property + def client_key(self) -> str: + """Client key path for TLS.""" + return "/certs/client-key.pem" if self.docker_sock.startswith("tcps://") else "" + + @property + def client_cert(self) -> str: + """Client cert path for TLS.""" + return "/certs/client-cert.pem" if self.docker_sock.startswith("tcps://") else "" \ No newline at end of file diff --git a/autoheal/docker_api.py b/autoheal/docker_api.py new file mode 100644 index 0000000..02c4bfe --- /dev/null +++ b/autoheal/docker_api.py @@ -0,0 +1,60 @@ +"""Docker API interaction functions.""" + +import docker +from .config import Config +from .logging_utils import get_logger + +logger = get_logger() + + +def get_docker_client(config: Config) -> docker.DockerClient: + """Get Docker client.""" + kwargs = {} + if config.unix_sock: + kwargs["base_url"] = f"unix://{config.unix_sock}" + elif config.docker_sock.startswith("tcp://"): + kwargs["base_url"] = config.docker_sock + elif config.docker_sock.startswith("tcps://"): + tls_config = docker.tls.TLSConfig( + ca_cert=config.ca_cert, + client_cert=(config.client_cert, config.client_key), + verify=True + ) + kwargs["base_url"] = config.docker_sock + kwargs["tls"] = tls_config + else: + pass # default + return docker.DockerClient(**kwargs) + + +def get_container_info(config: Config) -> list: + """Get list of unhealthy containers.""" + client = get_docker_client(config) + filters = {"health": ["unhealthy"]} + if config.autoheal_container_label != "all": + filters["label"] = [f"{config.autoheal_container_label}=true"] + if config.autoheal_only_monitor_running: + filters["status"] = ["running"] + return client.containers.list(filters=filters) + + +def restart_container(container_id: str, timeout: int, config: Config) -> bool: + """Restart a container.""" + client = get_docker_client(config) + try: + container = client.containers.get(container_id) + container.restart(timeout=timeout) + return True + except docker.errors.APIError: + return False + + +def stop_container(container_id: str, timeout: int, config: Config) -> bool: + """Stop a container.""" + client = get_docker_client(config) + try: + container = client.containers.get(container_id) + container.stop(timeout=timeout) + return True + except docker.errors.APIError: + return False \ No newline at end of file diff --git a/autoheal/health_monitor.py b/autoheal/health_monitor.py new file mode 100644 index 0000000..1ce3fec --- /dev/null +++ b/autoheal/health_monitor.py @@ -0,0 +1,61 @@ +"""Health monitoring logic.""" + +import time +from .config import Config +from .docker_api import get_container_info, restart_container, stop_container +from .restart_tracker import RestartTracker +from .notifications import notify_webhook, notify_apprise, notify_post_restart_script +from .logging_utils import get_logger + +logger = get_logger() + + +def monitor_containers(config: Config): + """Main monitoring loop.""" + tracker = RestartTracker(config) + + logger.info("Starting container health monitoring") + + while True: + containers = get_container_info(config) + + for container in containers: + if container.labels.get("autoheal") == "False": + continue + + container_id = container.id + container_name = container.name + container_state = container.status + stop_timeout = container.labels.get("autoheal.stop.timeout", config.autoheal_default_stop_timeout) + + short_id = container_id[:12] + + if not container_name: + logger.warning(f"Container name of ({short_id}) is null - don't restart") + continue + + if container_state == "restarting": + logger.info(f"Container {container_name} ({short_id}) found to be restarting - don't restart") + continue + + if tracker.should_stop_container(container_id): + logger.warning(f"Container {container_name} ({short_id}) exceeded restart threshold - Stopping") + if stop_container(container_id, int(stop_timeout), config): + logger.info(f"Successfully stopped container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) exceeded restart threshold. Stopped!", config) + else: + logger.error(f"Failed to stop container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) exceeded restart threshold. Failed to stop!", config) + else: + logger.info(f"Container {container_name} ({short_id}) found to be unhealthy - Restarting") + if restart_container(container_id, int(stop_timeout), config): + logger.info(f"Successfully restarted container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) found to be unhealthy. Restarted!", config) + else: + logger.error(f"Failed to restart container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) found to be unhealthy. Failed to restart!", config) + + notify_apprise(f"Container {container_name} ({short_id}) action taken", config) + notify_post_restart_script(container_name, short_id, container_state, int(stop_timeout), config) + + time.sleep(config.autoheal_interval) \ No newline at end of file diff --git a/autoheal/logging_utils.py b/autoheal/logging_utils.py new file mode 100644 index 0000000..8727049 --- /dev/null +++ b/autoheal/logging_utils.py @@ -0,0 +1,45 @@ +"""Logging utilities for Docker Autoheal.""" + +import logging +from datetime import datetime + + +class CustomFormatter(logging.Formatter): + """Custom formatter for log messages.""" + + def format(self, record): + timestamp = datetime.now().strftime("%I:%M:%S %p") + level = record.levelname + message = record.getMessage() + + if level == "DEBUG": + return f"[{timestamp}] 🔍 DEBUG: {message}" + elif level == "INFO": + return f"[{timestamp}] ℹī¸ INFO: {message}" + elif level == "WARNING": + return f"[{timestamp}] ⚠ī¸ WARN: {message}" + elif level == "ERROR": + return f"[{timestamp}] ❌ ERROR: {message}" + return f"[{timestamp}] {level}: {message}" + + +_logger = None + + +def get_logger() -> logging.Logger: + """Get the logger, setting it up if not already.""" + global _logger + if _logger is None: + _logger = logging.getLogger("autoheal") + _logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setFormatter(CustomFormatter()) + _logger.addHandler(handler) + return _logger + + +def setup_logging(log_level: int) -> logging.Logger: + """Set up logging level.""" + logger = get_logger() + logger.setLevel(logging.DEBUG if log_level == 2 else logging.INFO) + return logger \ No newline at end of file diff --git a/autoheal/notifications.py b/autoheal/notifications.py new file mode 100644 index 0000000..eeb6657 --- /dev/null +++ b/autoheal/notifications.py @@ -0,0 +1,47 @@ +"""Notification handling for webhooks and Apprise.""" + +import json +import requests +from .config import Config +from .logging_utils import get_logger + +logger = get_logger() + + +def notify_webhook(message: str, config: Config): + """Send webhook notification.""" + if not config.webhook_url: + return + + payload = {config.webhook_json_entry: message} + try: + requests.post(config.webhook_url, json=payload, timeout=30) + logger.debug("Sent webhook notification") + except requests.RequestException as e: + logger.error(f"Failed to send webhook: {e}") + + +def notify_apprise(message: str, config: Config): + """Send Apprise notification.""" + if not config.apprise_url: + return + + payload = {"title": "Autoheal", "body": message} + try: + requests.post(config.apprise_url, json=payload, timeout=30) + logger.debug("Sent Apprise notification") + except requests.RequestException as e: + logger.error(f"Failed to send Apprise: {e}") + + +def notify_post_restart_script(container_name: str, container_short_id: str, container_state: str, timeout: int, config: Config): + """Execute post-restart script.""" + if not config.post_restart_script: + return + + import subprocess + try: + subprocess.run([config.post_restart_script, container_name, container_short_id, container_state, str(timeout)], check=True) + logger.debug("Executed post-restart script") + except subprocess.CalledProcessError as e: + logger.error(f"Post-restart script failed: {e}") \ No newline at end of file diff --git a/autoheal/restart_tracker.py b/autoheal/restart_tracker.py new file mode 100644 index 0000000..9982848 --- /dev/null +++ b/autoheal/restart_tracker.py @@ -0,0 +1,57 @@ +"""Restart tracking for containers.""" + +import os +import time +from .config import Config + + +class RestartTracker: + """Tracks restart counts and timestamps for containers.""" + + def __init__(self, config: Config): + self.config = config + self.tracker_file = "/tmp/autoheal_restart_tracker.txt" + if not os.path.exists(self.tracker_file): + with open(self.tracker_file, "w") as f: + pass + + def get_restart_info(self, container_id: str) -> tuple[int, int]: + """Get first restart time and count for a container.""" + with open(self.tracker_file, "r") as f: + for line in f: + parts = line.strip().split() + if len(parts) >= 3 and parts[0] == container_id: + return int(parts[1]), int(parts[2]) + return int(time.time()), 0 + + def update_restart_info(self, container_id: str, first_time: int, count: int): + """Update restart info for a container.""" + lines = [] + with open(self.tracker_file, "r") as f: + lines = f.readlines() + + # Remove existing entry + lines = [line for line in lines if not line.startswith(container_id + " ")] + + # Add new entry + lines.append(f"{container_id} {first_time} {count}\n") + + with open(self.tracker_file, "w") as f: + f.writelines(lines) + + def should_stop_container(self, container_id: str) -> bool: + """Check if container should be stopped based on restart threshold.""" + current_time = int(time.time()) + first_time, count = self.get_restart_info(container_id) + + if current_time - first_time > self.config.autoheal_restart_window: + first_time = current_time + count = 0 + + count += 1 + + if count >= self.config.autoheal_restart_threshold: + return True + + self.update_restart_info(container_id, first_time, count) + return False \ No newline at end of file diff --git a/docker-entrypoint b/docker-entrypoint deleted file mode 100755 index e61c161..0000000 --- a/docker-entrypoint +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env sh - -set -e -# shellcheck disable=2039 -set -o pipefail - -VERSION="1.3.0" - -# LOG_LEVEL: 1=Normal (INFO/WARN/ERROR only), 2=Debug (all messages) -LOG_LEVEL=${LOG_LEVEL:-1} - -log() { - local level=$1 - local message=$2 - local timestamp - timestamp=$(date '+%I:%M:%S %p') # 12-hour format with AM/PM - - case $level in - "DEBUG") - [ "$LOG_LEVEL" -eq 2 ] && echo "[$timestamp] 🔍 DEBUG: $message" - ;; - "INFO") - echo "[$timestamp] ℹī¸ INFO: $message" - ;; - "WARN") - echo "[$timestamp] ⚠ī¸ WARN: $message" >&2 - ;; - "ERROR") - echo "[$timestamp] ❌ ERROR: $message" >&2 - ;; - esac -} - -DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock} -UNIX_SOCK="" -CURL_TIMEOUT=${CURL_TIMEOUT:-30} -WEBHOOK_URL=${WEBHOOK_URL:-""} -WEBHOOK_JSON_KEY=${WEBHOOK_JSON_KEY:-"text"} -APPRISE_URL=${APPRISE_URL:-""} - -# only use unix domain socket if no TCP endpoint is defined -case "${DOCKER_SOCK}" in - "tcp://"*) HTTP_ENDPOINT="$(echo "${DOCKER_SOCK}" | sed 's#tcp://#http://#')" - ;; - "tcps://"*) HTTP_ENDPOINT="$(echo "${DOCKER_SOCK}" | sed 's#tcps://#https://#')" - CA="--cacert /certs/ca.pem" - CLIENT_KEY="--key /certs/client-key.pem" - CLIENT_CERT="--cert /certs/client-cert.pem" - ;; - *) HTTP_ENDPOINT="http://localhost" - UNIX_SOCK="--unix-socket ${DOCKER_SOCK}" - ;; -esac - -AUTOHEAL_CONTAINER_LABEL=${AUTOHEAL_CONTAINER_LABEL:-autoheal} -AUTOHEAL_START_PERIOD=${AUTOHEAL_START_PERIOD:-0} -AUTOHEAL_INTERVAL=${AUTOHEAL_INTERVAL:-5} -AUTOHEAL_DEFAULT_STOP_TIMEOUT=${AUTOHEAL_DEFAULT_STOP_TIMEOUT:-10} -AUTOHEAL_ONLY_MONITOR_RUNNING=${AUTOHEAL_ONLY_MONITOR_RUNNING:-false} -AUTOHEAL_RESTART_THRESHOLD=${AUTOHEAL_RESTART_THRESHOLD:-5} -AUTOHEAL_RESTART_WINDOW=${AUTOHEAL_RESTART_WINDOW:-600} - -log "INFO" "Docker Autoheal v${VERSION} starting up" -log "INFO" "Configuration:" -log "INFO" " AUTOHEAL_CONTAINER_LABEL=${AUTOHEAL_CONTAINER_LABEL}" -log "INFO" " AUTOHEAL_START_PERIOD=${AUTOHEAL_START_PERIOD}" -log "INFO" " AUTOHEAL_INTERVAL=${AUTOHEAL_INTERVAL}" -log "INFO" " AUTOHEAL_DEFAULT_STOP_TIMEOUT=${AUTOHEAL_DEFAULT_STOP_TIMEOUT}" -log "INFO" " AUTOHEAL_ONLY_MONITOR_RUNNING=${AUTOHEAL_ONLY_MONITOR_RUNNING}" -log "INFO" " AUTOHEAL_RESTART_THRESHOLD=${AUTOHEAL_RESTART_THRESHOLD}" -log "INFO" " AUTOHEAL_RESTART_WINDOW=${AUTOHEAL_RESTART_WINDOW}" - -# Temporary file to store restart counts and timestamps -RESTART_TRACKER_FILE="/tmp/autoheal_restart_tracker.txt" -touch "$RESTART_TRACKER_FILE" - -docker_curl() { - curl --max-time "${CURL_TIMEOUT}" --no-buffer -s \ - ${CA} ${CLIENT_KEY} ${CLIENT_CERT} \ - ${UNIX_SOCK} \ - "$@" -} - -# shellcheck disable=2039 -get_container_info() { - local label_filter - local running_filter - local url - - # Set container selector - if [ "$AUTOHEAL_CONTAINER_LABEL" = "all" ] - then - label_filter="" - else - label_filter=",\"label\":\[\"${AUTOHEAL_CONTAINER_LABEL}=true\"\]" - fi - if [ "$AUTOHEAL_ONLY_MONITOR_RUNNING" = false ] - then - running_filter="" - else - running_filter=",\"status\":\[\"running\"\]" - fi - url="${HTTP_ENDPOINT}/containers/json?filters=\{\"health\":\[\"unhealthy\"\]${label_filter}${running_filter}\}" - docker_curl "$url" -} - -# shellcheck disable=2039 -restart_container() { - local container_id="$1" - local timeout="$2" - - docker_curl -f -X POST "${HTTP_ENDPOINT}/containers/${container_id}/restart?t=${timeout}" -} - -# shellcheck disable=2039 -stop_container() { - local container_id="$1" - local timeout="$2" - - docker_curl -f -X POST "${HTTP_ENDPOINT}/containers/${container_id}/stop?t=${timeout}" -} - -notify_webhook() { - local text="$@" - - if [ -n "$WEBHOOK_URL" ] - then - log "DEBUG" "Sending webhook notification" - # execute webhook requests as background process to prevent healer from blocking - curl -s -X POST -H "Content-type: application/json" -d "$(generate_webhook_payload "$text")" "$WEBHOOK_URL" - fi - - if [ -n "$APPRISE_URL" ] - then - log "DEBUG" "Sending Apprise notification" - # execute webhook requests as background process to prevent healer from blocking - curl -s -X POST -H "Content-type: application/json" -d "$(generate_apprise_payload "$text")" "$APPRISE_URL" - fi -} - -notify_post_restart_script() { - if [ -n "$POST_RESTART_SCRIPT" ] - then - log "DEBUG" "Executing post-restart script" - # execute post restart script as background process to prevent healer from blocking - $POST_RESTART_SCRIPT "$@" & - fi -} - -generate_webhook_payload() { - local text="$@" - cat < "${RESTART_TRACKER_FILE}.tmp" || true - mv "${RESTART_TRACKER_FILE}.tmp" "$RESTART_TRACKER_FILE" - echo "$CONTAINER_ID $FIRST_RESTART_TIME $RESTART_COUNT" >> "$RESTART_TRACKER_FILE" - - if [ $RESTART_COUNT -ge $AUTOHEAL_RESTART_THRESHOLD ] - then - log "WARN" "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) has been restarted $RESTART_COUNT times within the last $AUTOHEAL_RESTART_WINDOW seconds - Stopping container now with ${TIMEOUT}s timeout" - if ! stop_container "$CONTAINER_ID" "$TIMEOUT" - then - log "ERROR" "Failed to stop container $CONTAINER_NAME (${CONTAINER_SHORT_ID})" - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) has been restarted $RESTART_COUNT times within the last $AUTOHEAL_RESTART_WINDOW seconds. Failed to stop the container!" & - else - log "INFO" "Successfully stopped container $CONTAINER_NAME (${CONTAINER_SHORT_ID})" - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) has been restarted $RESTART_COUNT times within the last $AUTOHEAL_RESTART_WINDOW seconds. Successfully stopped the container!" & - fi - notify_post_restart_script "$CONTAINER_NAME" "$CONTAINER_SHORT_ID" "$CONTAINER_STATE" "$TIMEOUT" & - else - log "INFO" "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy - Restarting container now with ${TIMEOUT}s timeout" - if ! restart_container "$CONTAINER_ID" "$TIMEOUT" - then - log "ERROR" "Failed to restart container $CONTAINER_NAME (${CONTAINER_SHORT_ID})" - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" & - else - log "INFO" "Successfully restarted container $CONTAINER_NAME (${CONTAINER_SHORT_ID})" - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" & - fi - notify_post_restart_script "$CONTAINER_NAME" "$CONTAINER_SHORT_ID" "$CONTAINER_STATE" "$TIMEOUT" & - fi - fi - done - sleep "$AUTOHEAL_INTERVAL" & - wait $! - done -else - exec "$@" -fi diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..828fb21 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "docker-autoheal" +version = "2.0.0" +description = "Monitor and automatically restart unhealthy Docker containers" +authors = [{name = "JourneyOver"}] +dependencies = [ + "requests", + "docker" +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["autoheal"] \ No newline at end of file diff --git a/tests/.env b/tests/.env deleted file mode 100755 index 74a35dc..0000000 --- a/tests/.env +++ /dev/null @@ -1 +0,0 @@ -AUTOHEAL_CONTAINER_LABEL=autoheal-test \ No newline at end of file diff --git a/tests/README.md b/tests/README.md deleted file mode 100755 index 92932ee..0000000 --- a/tests/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Docker Autoheal Tests - -Docker Compose is used to build and deploy test environment. - -test.sh waits on watch-autoheal exit code. - -Currently setup to a very basic exit 1 on invalid restart and exit 0 on valid restart. - -## Run tests -``` -cd tests -./tests.sh -``` - -## Run tests in CI -``` -cd tests -export "AUTOHEAL_CONTAINER_LABEL=autoheal-123456" -./tests.sh "MY_UNIQUE_BUILD_NUMBER_123456" -``` - -This enables the tests to only restart containers within the test spec by using -unique docker-compose project names and autoheal labels (as long as you replace -123456 by a unique number) diff --git a/tests/docker-compose.autoheal.yml b/tests/docker-compose.autoheal.yml deleted file mode 100755 index d5c72b9..0000000 --- a/tests/docker-compose.autoheal.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3.7' - -services: - - watch-autoheal: - build: watch-autoheal - restart: "no" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - environment: - COMPOSE_PROJECT_NAME: $COMPOSE_PROJECT_NAME - network_mode: none diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml deleted file mode 100755 index ca0a152..0000000 --- a/tests/docker-compose.yml +++ /dev/null @@ -1,57 +0,0 @@ -version: '3.7' - -services: - - should-keep-restarting: - # this container should be restarted by autoheal because its unhealthy and has the autoheal label - image: alpine - network_mode: none - restart: "no" - labels: - - "$AUTOHEAL_CONTAINER_LABEL=true" - healthcheck: - test: exit 1 - interval: 3s - timeout: 1s - retries: 3 - start_period: 5s - command: tail -f /dev/null - - shouldnt-restart-healthy: - # this container shouldn't be restarted by autoheal because its healthy - image: alpine - network_mode: none - restart: "no" - labels: - - "$AUTOHEAL_CONTAINER_LABEL=true" - healthcheck: - test: exit 0 - interval: 2s - timeout: 1s - retries: 1 - start_period: 1s - command: tail -f /dev/null - - shouldnt-restart-no-label: - # this container shouldn't be restarted by autoheal because its missing the autoheal label - image: alpine - network_mode: none - restart: "no" - healthcheck: - test: exit 1 - interval: 3s - timeout: 1s - retries: 1 - start_period: 5s - command: tail -f /dev/null - - autoheal: - build: - context: ../ - restart: unless-stopped - environment: - AUTOHEAL_CONTAINER_LABEL: "${AUTOHEAL_CONTAINER_LABEL:-all}" - AUTOHEAL_INTERVAL: "10" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - network_mode: none diff --git a/tests/tests.sh b/tests/tests.sh deleted file mode 100755 index 703e3a5..0000000 --- a/tests/tests.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -COMPOSE_PROJECT_NAME=${1:-autoheal-test} -export COMPOSE_PROJECT_NAME - -function cleanup() -{ - exit_status=$? - echo "exit was $exit_status" - # stop autoheal first, to stop it restarting the test containers while we try to stop them - docker-compose stop autoheal - docker-compose -f docker-compose.autoheal.yml -f docker-compose.yml down || true - exit "$exit_status" -} -trap cleanup EXIT -docker-compose up --build -d -docker-compose -f docker-compose.autoheal.yml up --build --exit-code-from watch-autoheal watch-autoheal - diff --git a/tests/watch-autoheal/Dockerfile b/tests/watch-autoheal/Dockerfile deleted file mode 100755 index 9f28f04..0000000 --- a/tests/watch-autoheal/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM alpine:latest - -RUN apk --update add bash docker - - -WORKDIR /app -COPY . . - -ENTRYPOINT ["/app/entrypoint.sh"] \ No newline at end of file diff --git a/tests/watch-autoheal/entrypoint.sh b/tests/watch-autoheal/entrypoint.sh deleted file mode 100755 index e411ff1..0000000 --- a/tests/watch-autoheal/entrypoint.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -listenToDockerEvents() -{ - local expected_restarts - local LOGLINE - expected_restarts=0 - docker events --filter 'com.docker.compose.service=should-keep-restarting' --filter 'com.docker.compose.service=shouldnt-restart-*' --filter 'event=restart' | while read -r LOGLINE - do - echo "$LOGLINE" - # may be more elaborate checks here. - [[ $LOGLINE == *"container restart "*"com.docker.compose.service=shouldnt-restart-"* && $LOGLINE == *"com.docker.compose.project=$COMPOSE_PROJECT_NAME"* ]] && echo "ERR: No restarts expected on shouldnt-restart-* containers!" 1>&2 && pkill -9 docker && exit 1 - [[ $LOGLINE == *"container restart "*"com.docker.compose.service=should-keep-restarting"* && $LOGLINE == *"com.docker.compose.project=$COMPOSE_PROJECT_NAME"* ]] && echo "OK: Expected restart on should-keep-restarting container!" && pkill -9 docker && expected_restarts=$((expected_restarts + 1)) - [[ $expected_restarts == 1 ]] && echo "OK: All expected restarts happened" && exit 0 - done -} - -export -f listenToDockerEvents -timeout 60s bash -c listenToDockerEvents From efeb676f32090ce682f1e03e211542b6a35c8213 Mon Sep 17 00:00:00 2001 From: Journey Date: Mon, 24 Nov 2025 16:03:18 -0600 Subject: [PATCH 2/4] some more changes for the python variant --- autoheal/__main__.py | 15 ++++--- autoheal/docker_api.py | 40 +++++++++++++------ autoheal/health_monitor.py | 81 ++++++++++++++++++++++---------------- 3 files changed, 83 insertions(+), 53 deletions(-) diff --git a/autoheal/__main__.py b/autoheal/__main__.py index c4e9abb..7ede13d 100644 --- a/autoheal/__main__.py +++ b/autoheal/__main__.py @@ -4,15 +4,16 @@ import sys import time import os +import threading from .config import Config from .health_monitor import monitor_containers from .logging_utils import setup_logging -def signal_handler(signum, frame): - """Handle SIGTERM.""" - logger.info("Received SIGTERM signal, shutting down") - sys.exit(143) +def signal_handler(signum, frame, shutdown_event): + """Handle SIGTERM and SIGINT.""" + logger.info(f"Received signal {signum}, shutting down gracefully") + shutdown_event.set() def main(): @@ -38,13 +39,15 @@ def main(): logger.error("Unix socket is currently not available") sys.exit(1) - signal.signal(signal.SIGTERM, signal_handler) + shutdown_event = threading.Event() + signal.signal(signal.SIGTERM, lambda signum, frame: signal_handler(signum, frame, shutdown_event)) + signal.signal(signal.SIGINT, lambda signum, frame: signal_handler(signum, frame, shutdown_event)) if config.autoheal_start_period > 0: logger.info(f"Waiting {config.autoheal_start_period} second(s) before starting monitoring") time.sleep(config.autoheal_start_period) - monitor_containers(config) + monitor_containers(config, shutdown_event) if __name__ == "__main__": diff --git a/autoheal/docker_api.py b/autoheal/docker_api.py index 02c4bfe..8543174 100644 --- a/autoheal/docker_api.py +++ b/autoheal/docker_api.py @@ -24,37 +24,53 @@ def get_docker_client(config: Config) -> docker.DockerClient: kwargs["tls"] = tls_config else: pass # default - return docker.DockerClient(**kwargs) + try: + return docker.DockerClient(**kwargs) + except Exception as e: + logger.error(f"Failed to create Docker client: {e}") + raise def get_container_info(config: Config) -> list: """Get list of unhealthy containers.""" - client = get_docker_client(config) - filters = {"health": ["unhealthy"]} - if config.autoheal_container_label != "all": - filters["label"] = [f"{config.autoheal_container_label}=true"] - if config.autoheal_only_monitor_running: - filters["status"] = ["running"] - return client.containers.list(filters=filters) + try: + client = get_docker_client(config) + filters = {"health": ["unhealthy"]} + if config.autoheal_container_label != "all": + filters["label"] = [f"{config.autoheal_container_label}=true"] + if config.autoheal_only_monitor_running: + filters["status"] = ["running"] + return client.containers.list(filters=filters) + except Exception as e: + logger.error(f"Failed to get container info: {e}") + return [] def restart_container(container_id: str, timeout: int, config: Config) -> bool: """Restart a container.""" - client = get_docker_client(config) try: + client = get_docker_client(config) container = client.containers.get(container_id) container.restart(timeout=timeout) return True - except docker.errors.APIError: + except docker.errors.APIError as e: + logger.error(f"Failed to restart container {container_id}: {e}") + return False + except Exception as e: + logger.error(f"Unexpected error restarting container {container_id}: {e}") return False def stop_container(container_id: str, timeout: int, config: Config) -> bool: """Stop a container.""" - client = get_docker_client(config) try: + client = get_docker_client(config) container = client.containers.get(container_id) container.stop(timeout=timeout) return True - except docker.errors.APIError: + except docker.errors.APIError as e: + logger.error(f"Failed to stop container {container_id}: {e}") + return False + except Exception as e: + logger.error(f"Unexpected error stopping container {container_id}: {e}") return False \ No newline at end of file diff --git a/autoheal/health_monitor.py b/autoheal/health_monitor.py index 1ce3fec..4ac4d2c 100644 --- a/autoheal/health_monitor.py +++ b/autoheal/health_monitor.py @@ -10,52 +10,63 @@ logger = get_logger() -def monitor_containers(config: Config): +def monitor_containers(config: Config, shutdown_event: threading.Event): """Main monitoring loop.""" tracker = RestartTracker(config) logger.info("Starting container health monitoring") - while True: - containers = get_container_info(config) + while not shutdown_event.is_set(): + try: + containers = get_container_info(config) - for container in containers: - if container.labels.get("autoheal") == "False": - continue + for container in containers: + if shutdown_event.is_set(): + break + if container.labels.get("autoheal") == "False": + continue - container_id = container.id - container_name = container.name - container_state = container.status - stop_timeout = container.labels.get("autoheal.stop.timeout", config.autoheal_default_stop_timeout) + container_id = container.id + container_name = container.name + container_state = container.status + stop_timeout = container.labels.get("autoheal.stop.timeout", config.autoheal_default_stop_timeout) - short_id = container_id[:12] + short_id = container_id[:12] - if not container_name: - logger.warning(f"Container name of ({short_id}) is null - don't restart") - continue + if not container_name: + logger.warning(f"Container name of ({short_id}) is null - don't restart") + continue - if container_state == "restarting": - logger.info(f"Container {container_name} ({short_id}) found to be restarting - don't restart") - continue + if container_state == "restarting": + logger.info(f"Container {container_name} ({short_id}) found to be restarting - don't restart") + continue - if tracker.should_stop_container(container_id): - logger.warning(f"Container {container_name} ({short_id}) exceeded restart threshold - Stopping") - if stop_container(container_id, int(stop_timeout), config): - logger.info(f"Successfully stopped container {container_name} ({short_id})") - notify_webhook(f"Container {container_name} ({short_id}) exceeded restart threshold. Stopped!", config) + if tracker.should_stop_container(container_id): + logger.warning(f"Container {container_name} ({short_id}) exceeded restart threshold - Stopping") + if stop_container(container_id, int(stop_timeout), config): + logger.info(f"Successfully stopped container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) exceeded restart threshold. Stopped!", config) + else: + logger.error(f"Failed to stop container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) exceeded restart threshold. Failed to stop!", config) else: - logger.error(f"Failed to stop container {container_name} ({short_id})") - notify_webhook(f"Container {container_name} ({short_id}) exceeded restart threshold. Failed to stop!", config) - else: - logger.info(f"Container {container_name} ({short_id}) found to be unhealthy - Restarting") - if restart_container(container_id, int(stop_timeout), config): - logger.info(f"Successfully restarted container {container_name} ({short_id})") - notify_webhook(f"Container {container_name} ({short_id}) found to be unhealthy. Restarted!", config) - else: - logger.error(f"Failed to restart container {container_name} ({short_id})") - notify_webhook(f"Container {container_name} ({short_id}) found to be unhealthy. Failed to restart!", config) + logger.info(f"Container {container_name} ({short_id}) found to be unhealthy - Restarting") + if restart_container(container_id, int(stop_timeout), config): + logger.info(f"Successfully restarted container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) found to be unhealthy. Restarted!", config) + else: + logger.error(f"Failed to restart container {container_name} ({short_id})") + notify_webhook(f"Container {container_name} ({short_id}) found to be unhealthy. Failed to restart!", config) + + notify_apprise(f"Container {container_name} ({short_id}) action taken", config) + notify_post_restart_script(container_name, short_id, container_state, int(stop_timeout), config) + + except Exception as e: + logger.error(f"Unexpected error in monitoring loop: {e}") - notify_apprise(f"Container {container_name} ({short_id}) action taken", config) - notify_post_restart_script(container_name, short_id, container_state, int(stop_timeout), config) + if not shutdown_event.wait(config.autoheal_interval): + continue # timeout, continue loop + else: + break # event set - time.sleep(config.autoheal_interval) \ No newline at end of file + logger.info("Container health monitoring stopped") \ No newline at end of file From ccda994f4b47c1979cd266ee6eabe8096a203bb0 Mon Sep 17 00:00:00 2001 From: Journey Date: Tue, 25 Nov 2025 02:16:59 -0600 Subject: [PATCH 3/4] Hopefully fix an error during shutdown. --- autoheal/__main__.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/autoheal/__main__.py b/autoheal/__main__.py index 7ede13d..af29fe0 100644 --- a/autoheal/__main__.py +++ b/autoheal/__main__.py @@ -8,19 +8,16 @@ from .config import Config from .health_monitor import monitor_containers from .logging_utils import setup_logging - - -def signal_handler(signum, frame, shutdown_event): - """Handle SIGTERM and SIGINT.""" - logger.info(f"Received signal {signum}, shutting down gracefully") - shutdown_event.set() - - def main(): """Main function.""" config = Config() logger = setup_logging(config.log_level) + def signal_handler(signum, frame, shutdown_event): + """Handle SIGTERM and SIGINT.""" + logger.info(f"Received signal {signum}, shutting down gracefully") + shutdown_event.set() + logger.info(f"Docker Autoheal v{__import__('autoheal').__version__} starting up") logger.info("Configuration:") for key, value in config.__dict__.items(): From e758835a86e0f797c5d767da1a28b07662f85092 Mon Sep 17 00:00:00 2001 From: Journey Date: Wed, 3 Dec 2025 14:07:23 -0600 Subject: [PATCH 4/4] chore(deps): update python docker tag to v3.14.1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c96d80b..58f900f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.14.0-alpine +FROM python:3.14.1-alpine # Install required packages RUN apk add --no-cache curl tzdata procps