From 643d94ec9f6a70e3bed95396516c4e018a399aa4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 21:31:52 +0000 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20auto-heal=20ERR=5FCONNECTION=5FREFUS?= =?UTF-8?q?ED=20=E2=80=94=20keepalive=20every=205min,=20nginx=20watchdog,?= =?UTF-8?q?=20server=20cron,=20faster=20deploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/ruffy80/ZeusAI/sessions/f66ac24f-d661-401f-ad82-a02b095d6f10 Co-authored-by: ruffy80 <29306714+ruffy80@users.noreply.github.com> --- .github/workflows/unicorn-keepalive.yml | 15 +++++- .github/workflows/vercel-deploy.yml | 20 +++++--- UNICORN_FINAL/scripts/health-guardian.js | 60 ++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unicorn-keepalive.yml b/.github/workflows/unicorn-keepalive.yml index c51da1f1..3f7dfe4c 100644 --- a/.github/workflows/unicorn-keepalive.yml +++ b/.github/workflows/unicorn-keepalive.yml @@ -5,7 +5,7 @@ name: ๐Ÿ”— Unicorn Keep-Alive (Hetzner Health Check) on: schedule: - - cron: '*/30 * * * *' # every 30 minutes + - cron: '*/5 * * * *' # every 5 minutes workflow_dispatch: inputs: force_redeploy: @@ -188,6 +188,19 @@ jobs: # Verify nginx port 80 is accessible locally curl -sf --max-time 5 http://localhost:80/ >/dev/null 2>&1 && echo "โœ… Nginx port 80 responding" || echo "โš ๏ธ Nginx port 80 not responding โ€” check: journalctl -u nginx -n 20" curl -kfsS --resolve "$DOMAIN:443:127.0.0.1" "https://$DOMAIN/health" >/dev/null 2>&1 && echo "โœ… Nginx HTTPS responding" || echo "โš ๏ธ Nginx HTTPS still not responding โ€” check certbot/nginx logs" + + # โ”€โ”€ Ensure server-side self-healing cron exists โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + if [ -f "${DEPLOY_PATH}/scripts/fix-server.sh" ] && [ ! -f /etc/cron.d/unicorn-self-heal ]; then + mkdir -p "${DEPLOY_PATH}/logs" + cat > /etc/cron.d/unicorn-self-heal <> "${DEPLOY_PATH}/logs/fix-server-cron.log" 2>&1 +CRONEOF + chmod 644 /etc/cron.d/unicorn-self-heal + echo "โœ… Self-healing cron installed" + fi ENDSSH - name: Verify platform recovered after SSH restart diff --git a/.github/workflows/vercel-deploy.yml b/.github/workflows/vercel-deploy.yml index 053d551a..fd28b2e5 100644 --- a/.github/workflows/vercel-deploy.yml +++ b/.github/workflows/vercel-deploy.yml @@ -485,12 +485,7 @@ jobs: shopt -u nullglob fi - # โ”€โ”€ Stop PM2 before npm install to prevent ENOTEMPTY file locks โ”€โ”€ - if command -v pm2 >/dev/null 2>&1; then - pm2 stop unicorn 2>/dev/null || pm2 stop all 2>/dev/null || true - fi - - # โ”€โ”€ Install dependencies (with retry on ENOTEMPTY) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # โ”€โ”€ Install dependencies BEFORE stopping PM2 (reduce downtime) โ”€ npm install --legacy-peer-deps --no-audit --no-fund || { echo "โš ๏ธ npm install failed, cleaning and retrying..." npm cache clean --force 2>/dev/null || true @@ -685,6 +680,19 @@ jobs: bash scripts/setup-systemd.sh "${{ secrets.HETZNER_DEPLOY_PATH || '/opt/unicorn' }}" || true echo "โœ… Systemd auto-boot configured" fi + + # โ”€โ”€ Install server-side self-healing cron (every 5 min) โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + _CRON_DEPLOY="${{ secrets.HETZNER_DEPLOY_PATH || '/opt/unicorn' }}" + _CRON_DOMAIN="${{ secrets.SITE_DOMAIN || 'zeusai.pro' }}" + mkdir -p "${_CRON_DEPLOY}/logs" + cat > /etc/cron.d/unicorn-self-heal <> "${_CRON_DEPLOY}/logs/fix-server-cron.log" 2>&1 +CRONEOF + chmod 644 /etc/cron.d/unicorn-self-heal + echo "โœ… Self-healing cron installed (/etc/cron.d/unicorn-self-heal, every 5 min)" ENDSSH - name: Health check Hetzner backend diff --git a/UNICORN_FINAL/scripts/health-guardian.js b/UNICORN_FINAL/scripts/health-guardian.js index ecc3d0e8..8de023d4 100644 --- a/UNICORN_FINAL/scripts/health-guardian.js +++ b/UNICORN_FINAL/scripts/health-guardian.js @@ -16,10 +16,13 @@ const EXTERNAL_INTERVAL_MS = Math.max(parseInt(process.env.HEALTH_GUARDIAN_EXTER const FAIL_THRESHOLD = Math.max(parseInt(process.env.HEALTH_GUARDIAN_FAIL_THRESHOLD || '3', 10), 1); const HEAL_CMD = process.env.HEALTH_GUARDIAN_HEAL_CMD || 'systemctl restart unicorn'; const ROLLBACK_CMD = process.env.HEALTH_GUARDIAN_ROLLBACK_CMD || ''; +// Nginx check interval (default every 60s) +const NGINX_INTERVAL_MS = Math.max(parseInt(process.env.HEALTH_GUARDIAN_NGINX_MS || '60000', 10), 15000); let consecutiveFailures = 0; let consecutiveIntegrityFailures = 0; let consecutiveExternalFailures = 0; +let consecutiveNginxFailures = 0; function log(msg, extra) { const ts = new Date().toISOString(); @@ -129,6 +132,38 @@ function heal() { }); } +/** Verificฤƒ dacฤƒ Nginx rฤƒspunde pe portul 80 local */ +function checkNginx() { + return new Promise((resolve) => { + const req = http.get('http://127.0.0.1:80/', (res) => { + // Orice rฤƒspuns HTTP (inclusiv 4xx/5xx) รฎnseamnฤƒ cฤƒ Nginx ruleazฤƒ + res.resume(); + resolve({ ok: true, status: res.statusCode }); + }); + req.on('error', (err) => resolve({ ok: false, reason: err.message })); + req.setTimeout(5000, () => { req.destroy(); resolve({ ok: false, reason: 'timeout' }); }); + }); +} + +/** Reporneศ™te Nginx dacฤƒ este oprit */ +function healNginx() { + return new Promise((resolve) => { + const cmd = [ + 'nginx -t 2>/dev/null && systemctl reload nginx 2>/dev/null', + 'systemctl restart nginx 2>/dev/null', + 'service nginx restart 2>/dev/null', + ].join(' || '); + exec(cmd, { timeout: 20000 }, (err, stdout, stderr) => { + if (err) { + log('nginx heal failed', { error: err.message, stderr: (stderr || '').slice(0, 200) }); + return resolve(false); + } + log('nginx restarted successfully', { stdout: (stdout || '').slice(0, 200) }); + resolve(true); + }); + }); +} + async function loop() { const [result, shield] = await Promise.all([checkHealth(), checkShield()]); if (result.ok) { @@ -211,3 +246,28 @@ if (EXTERNAL_URL) { setInterval(externalLoop, EXTERNAL_INTERVAL_MS); setTimeout(externalLoop, 10000); // primฤƒ verificare externฤƒ dupฤƒ 10s } + +// โ”€โ”€ Nginx watchdog โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +async function nginxLoop() { + const result = await checkNginx(); + if (result.ok) { + if (consecutiveNginxFailures > 0) { + log(`nginx restored after ${consecutiveNginxFailures} failure(s)`); + notifyOrchestrator('nginx_restored', {}); + } + consecutiveNginxFailures = 0; + return; + } + consecutiveNginxFailures += 1; + log(`nginx health check failed (${consecutiveNginxFailures}/${FAIL_THRESHOLD})`, { reason: result.reason }); + notifyOrchestrator('nginx_failure', { reason: result.reason, failures: consecutiveNginxFailures }); + + if (consecutiveNginxFailures >= FAIL_THRESHOLD) { + log('nginx down threshold reached โ€” attempting nginx restart'); + await healNginx(); + consecutiveNginxFailures = 0; + } +} + +setInterval(nginxLoop, NGINX_INTERVAL_MS); +setTimeout(nginxLoop, 15000); // primฤƒ verificare Nginx dupฤƒ 15s From c72ad426295c3ee3753c138894ef8638ddaefad2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 21:35:44 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20address=20code=20review=20=E2=80=94?= =?UTF-8?q?=20restore=20PM2=20stop=20order,=20remove=20stderr=20suppressio?= =?UTF-8?q?n=20in=20nginx=20heal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/ruffy80/ZeusAI/sessions/f66ac24f-d661-401f-ad82-a02b095d6f10 Co-authored-by: ruffy80 <29306714+ruffy80@users.noreply.github.com> --- .github/workflows/vercel-deploy.yml | 6 +++++- UNICORN_FINAL/scripts/health-guardian.js | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vercel-deploy.yml b/.github/workflows/vercel-deploy.yml index fd28b2e5..3f5ac166 100644 --- a/.github/workflows/vercel-deploy.yml +++ b/.github/workflows/vercel-deploy.yml @@ -485,7 +485,11 @@ jobs: shopt -u nullglob fi - # โ”€โ”€ Install dependencies BEFORE stopping PM2 (reduce downtime) โ”€ + # โ”€โ”€ Stop PM2, install deps, restart immediately (minimize downtime) โ”€ + if command -v pm2 >/dev/null 2>&1; then + pm2 stop unicorn 2>/dev/null || pm2 stop all 2>/dev/null || true + fi + npm install --legacy-peer-deps --no-audit --no-fund || { echo "โš ๏ธ npm install failed, cleaning and retrying..." npm cache clean --force 2>/dev/null || true diff --git a/UNICORN_FINAL/scripts/health-guardian.js b/UNICORN_FINAL/scripts/health-guardian.js index 8de023d4..75510851 100644 --- a/UNICORN_FINAL/scripts/health-guardian.js +++ b/UNICORN_FINAL/scripts/health-guardian.js @@ -149,9 +149,9 @@ function checkNginx() { function healNginx() { return new Promise((resolve) => { const cmd = [ - 'nginx -t 2>/dev/null && systemctl reload nginx 2>/dev/null', - 'systemctl restart nginx 2>/dev/null', - 'service nginx restart 2>/dev/null', + 'nginx -t && systemctl reload nginx', + 'systemctl restart nginx', + 'service nginx restart', ].join(' || '); exec(cmd, { timeout: 20000 }, (err, stdout, stderr) => { if (err) {