fix: auto-heal ERR_CONNECTION_REFUSED on zeusai.pro

.github/workflows/unicorn-keepalive.yml

@@ -5,7 +5,7 @@ name: 🔗 Unicorn Keep-Alive (Hetzner Health Check)
 
 on:
   schedule:
-    - cron: '*/30 * * * *'   # every 30 minutes
+    - cron: '*/5 * * * *'    # every 5 minutes
   workflow_dispatch:
     inputs:
       force_redeploy:
@@ -188,6 +188,19 @@ jobs:
             # Verify nginx port 80 is accessible locally
             curl -sf --max-time 5 http://localhost:80/ >/dev/null 2>&1 && echo "✅ Nginx port 80 responding" || echo "⚠️  Nginx port 80 not responding — check: journalctl -u nginx -n 20"
             curl -kfsS --resolve "$DOMAIN:443:127.0.0.1" "https://$DOMAIN/health" >/dev/null 2>&1 && echo "✅ Nginx HTTPS responding" || echo "⚠️  Nginx HTTPS still not responding — check certbot/nginx logs"
+
+            # ── Ensure server-side self-healing cron exists ───────────────
+            if [ -f "${DEPLOY_PATH}/scripts/fix-server.sh" ] && [ ! -f /etc/cron.d/unicorn-self-heal ]; then
+              mkdir -p "${DEPLOY_PATH}/logs"
+              cat > /etc/cron.d/unicorn-self-heal <<CRONEOF
+# Auto-generated by ZeusAI keepalive — heals Nginx/PM2/firewall every 5 minutes
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+*/5 * * * * root bash "${DEPLOY_PATH}/scripts/fix-server.sh" "${DEPLOY_PATH}" "${DOMAIN}" >> "${DEPLOY_PATH}/logs/fix-server-cron.log" 2>&1
+CRONEOF
+              chmod 644 /etc/cron.d/unicorn-self-heal
+              echo "✅ Self-healing cron installed"
+            fi
           ENDSSH
 
       - name: Verify platform recovered after SSH restart

.github/workflows/vercel-deploy.yml

@@ -485,12 +485,11 @@ jobs:
               shopt -u nullglob
             fi
 
-            # ── Stop PM2 before npm install to prevent ENOTEMPTY file locks ──
+            # ── Stop PM2, install deps, restart immediately (minimize downtime) ─
             if command -v pm2 >/dev/null 2>&1; then
               pm2 stop unicorn 2>/dev/null || pm2 stop all 2>/dev/null || true
             fi
 
-            # ── Install dependencies (with retry on ENOTEMPTY) ───────────
             npm install --legacy-peer-deps --no-audit --no-fund || {
               echo "⚠️  npm install failed, cleaning and retrying..."
               npm cache clean --force 2>/dev/null || true
@@ -685,6 +684,19 @@ jobs:
               bash scripts/setup-systemd.sh "${{ secrets.HETZNER_DEPLOY_PATH || '/opt/unicorn' }}" || true
               echo "✅ Systemd auto-boot configured"
             fi
+
+            # ── Install server-side self-healing cron (every 5 min) ───────
+            _CRON_DEPLOY="${{ secrets.HETZNER_DEPLOY_PATH || '/opt/unicorn' }}"
+            _CRON_DOMAIN="${{ secrets.SITE_DOMAIN || 'zeusai.pro' }}"
+            mkdir -p "${_CRON_DEPLOY}/logs"
+            cat > /etc/cron.d/unicorn-self-heal <<CRONEOF
+# Auto-generated by ZeusAI deploy — heals Nginx/PM2/firewall every 5 minutes
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+*/5 * * * * root bash "${_CRON_DEPLOY}/scripts/fix-server.sh" "${_CRON_DEPLOY}" "${_CRON_DOMAIN}" >> "${_CRON_DEPLOY}/logs/fix-server-cron.log" 2>&1
+CRONEOF
+            chmod 644 /etc/cron.d/unicorn-self-heal
+            echo "✅ Self-healing cron installed (/etc/cron.d/unicorn-self-heal, every 5 min)"
           ENDSSH
 
       - name: Health check Hetzner backend

UNICORN_FINAL/scripts/health-guardian.js

@@ -16,10 +16,13 @@ const EXTERNAL_INTERVAL_MS = Math.max(parseInt(process.env.HEALTH_GUARDIAN_EXTER
 const FAIL_THRESHOLD     = Math.max(parseInt(process.env.HEALTH_GUARDIAN_FAIL_THRESHOLD || '3', 10), 1);
 const HEAL_CMD           = process.env.HEALTH_GUARDIAN_HEAL_CMD     || 'systemctl restart unicorn';
 const ROLLBACK_CMD       = process.env.HEALTH_GUARDIAN_ROLLBACK_CMD || '';
+// Nginx check interval (default every 60s)
+const NGINX_INTERVAL_MS  = Math.max(parseInt(process.env.HEALTH_GUARDIAN_NGINX_MS || '60000', 10), 15000);
 
 let consecutiveFailures          = 0;
 let consecutiveIntegrityFailures = 0;
 let consecutiveExternalFailures  = 0;
+let consecutiveNginxFailures     = 0;
 
 function log(msg, extra) {
   const ts = new Date().toISOString();
@@ -129,6 +132,38 @@ function heal() {
   });
 }
 
+/** Verifică dacă Nginx răspunde pe portul 80 local */
+function checkNginx() {
+  return new Promise((resolve) => {
+    const req = http.get('http://127.0.0.1:80/', (res) => {
+      // Orice răspuns HTTP (inclusiv 4xx/5xx) înseamnă că Nginx rulează
+      res.resume();
+      resolve({ ok: true, status: res.statusCode });
+    });
+    req.on('error', (err) => resolve({ ok: false, reason: err.message }));
+    req.setTimeout(5000, () => { req.destroy(); resolve({ ok: false, reason: 'timeout' }); });
+  });
+}
+
+/** Repornește Nginx dacă este oprit */
+function healNginx() {
+  return new Promise((resolve) => {
+    const cmd = [
+      'nginx -t && systemctl reload nginx',
+      'systemctl restart nginx',
+      'service nginx restart',
+    ].join(' || ');
+    exec(cmd, { timeout: 20000 }, (err, stdout, stderr) => {
+      if (err) {
+        log('nginx heal failed', { error: err.message, stderr: (stderr || '').slice(0, 200) });
+        return resolve(false);
+      }
+      log('nginx restarted successfully', { stdout: (stdout || '').slice(0, 200) });
+      resolve(true);
+    });
+  });
+}
+
 async function loop() {
   const [result, shield] = await Promise.all([checkHealth(), checkShield()]);
   if (result.ok) {
@@ -211,3 +246,28 @@ if (EXTERNAL_URL) {
   setInterval(externalLoop, EXTERNAL_INTERVAL_MS);
   setTimeout(externalLoop, 10000); // primă verificare externă după 10s
 }
+
+// ── Nginx watchdog ────────────────────────────────────────────────────────────
+async function nginxLoop() {
+  const result = await checkNginx();
+  if (result.ok) {
+    if (consecutiveNginxFailures > 0) {
+      log(`nginx restored after ${consecutiveNginxFailures} failure(s)`);
+      notifyOrchestrator('nginx_restored', {});
+    }
+    consecutiveNginxFailures = 0;
+    return;
+  }
+  consecutiveNginxFailures += 1;
+  log(`nginx health check failed (${consecutiveNginxFailures}/${FAIL_THRESHOLD})`, { reason: result.reason });
+  notifyOrchestrator('nginx_failure', { reason: result.reason, failures: consecutiveNginxFailures });
+
+  if (consecutiveNginxFailures >= FAIL_THRESHOLD) {
+    log('nginx down threshold reached — attempting nginx restart');
+    await healNginx();
+    consecutiveNginxFailures = 0;
+  }
+}
+
+setInterval(nginxLoop, NGINX_INTERVAL_MS);
+setTimeout(nginxLoop, 15000); // primă verificare Nginx după 15s