🚀 Deploy: Alert 자동 재학습 제거 → 관리자 이메일 알림 + 로컬 학습 자동 배포 구성

.github/workflows/deploy-churn.yaml

@@ -0,0 +1,22 @@
+name: Deploy (churn)
+
+on:
+  repository_dispatch:
+    types:
+      - deploy-churn
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: kubectl 설정
+        run: |
+          mkdir -p $HOME/.kube
+          echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config
+
+      - name: model-api 파드 재시작
+        run: kubectl rollout restart deployment/model-api -n model-api
+
+      - name: model-api 배포 확인
+        run: kubectl rollout status deployment/model-api -n model-api

k8s/alert-bridge/deployment.yaml

@@ -19,12 +19,33 @@ spec:
           ports:
             - containerPort: 8080
           env:
-            - name: GITHUB_TOKEN
+            - name: SMTP_HOST
               valueFrom:
                 secretKeyRef:
-                  name: github-credentials
-                  key: token
-            - name: GITHUB_REPO_OWNER
-              value: simGPT
-            - name: GITHUB_REPO_NAME
-              value: sw-mlops
+                  name: smtp-credentials
+                  key: host
+            - name: SMTP_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: smtp-credentials
+                  key: port
+            - name: SMTP_USER
+              valueFrom:
+                secretKeyRef:
+                  name: smtp-credentials
+                  key: user
+            - name: SMTP_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: smtp-credentials
+                  key: password
+            - name: ALERT_MAIL_FROM
+              valueFrom:
+                secretKeyRef:
+                  name: smtp-credentials
+                  key: mail-from
+            - name: ALERT_MAIL_TO
+              valueFrom:
+                secretKeyRef:
+                  name: smtp-credentials
+                  key: mail-to

services/alert-bridge/main.py

@@ -1,36 +1,57 @@
 import os
-import httpx
+import smtplib
+import ssl
+from datetime import datetime, timezone
+from email.mime.text import MIMEText
 from fastapi import FastAPI, Request
 
 app = FastAPI(title="Alert Bridge")
 
-GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
-GITHUB_REPO_OWNER = os.getenv("GITHUB_REPO_OWNER")
-GITHUB_REPO_NAME = os.getenv("GITHUB_REPO_NAME")
+SMTP_HOST = os.getenv("SMTP_HOST")
+SMTP_PORT = int(os.getenv("SMTP_PORT", "465"))
+SMTP_USER = os.getenv("SMTP_USER")
+SMTP_PASSWORD = os.getenv("SMTP_PASSWORD")
+ALERT_MAIL_FROM = os.getenv("ALERT_MAIL_FROM")
+ALERT_MAIL_TO = os.getenv("ALERT_MAIL_TO")
 
-# Alertmanager에서 알림을 받아서 GitHub Actions로 전달하는 역할을 하는 api 엔드포인트(Alertmanager가 요청을 보냄)
+# 알림 내용을 이메일로 작성하는 함수
+def build_email_body(alerts: list) -> str:
+    lines = [f"[sw-mlops 알림] {len(alerts)}개의 이상 감지 - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}\n"]
+    for a in alerts:
+        labels = a.get("labels", {})
+        annotations = a.get("annotations", {})
+        lines.append(f"• Alert   : {labels.get('alertname', '-')}")
+        lines.append(f"  Severity: {labels.get('severity', '-')}")
+        lines.append(f"  Summary : {annotations.get('summary', '-')}")
+        lines.append(f"  Detail  : {annotations.get('description', '-')}\n")
+    return "\n".join(lines)
+
+# 이메일 전송하는 함수
+def send_email(subject: str, body: str):
+    msg = MIMEText(body, "plain", "utf-8")
+    msg["Subject"] = subject
+    msg["From"] = ALERT_MAIL_FROM
+    msg["To"] = ALERT_MAIL_TO
+
+    with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as server:
+        server.starttls(context=ssl.create_default_context())
+        server.login(SMTP_USER, SMTP_PASSWORD)
+        server.sendmail(ALERT_MAIL_FROM, ALERT_MAIL_TO, msg.as_string())
+
+
+# 알림 받아서 이메일 전송하는 api
 @app.post("/alert")
 async def receive_alert(request: Request):
     payload = await request.json()
 
-    # alertmanager에서 받은 알림 중에서 firing 상태인 알림만 필터링
     firing_alerts = [a for a in payload.get("alerts", []) if a["status"] == "firing"]
     if not firing_alerts:
-        return {"message": "no firing alerts"}
+        return {"message": "발생중인 알림이 없습니다."}
 
     alert_names = [a["labels"].get("alertname", "") for a in firing_alerts]
+    subject = f"[sw-mlops 알림] {', '.join(alert_names)}"
+    body = build_email_body(firing_alerts) # 알림 내용을 이메일 body로 작성
+
+    send_email(subject, body) # 이메일 전송
 
-    async with httpx.AsyncClient() as client:
-        response = await client.post(
-            f"https://api.github.com/repos/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/dispatches",
-            headers={
-                "Authorization": f"Bearer {GITHUB_TOKEN}",
-                "Accept": "application/vnd.github+json",
-            },
-            json={
-                "event_type": "model-retrain",
-                "client_payload": {"alerts": alert_names},
-            },
-        )
-
-    return {"status": response.status_code, "alerts": alert_names}
+    return {"message": "이메일이 전송되었습니다.", "alerts": alert_names}

services/alert-bridge/requirements.txt

@@ -1,3 +1,2 @@
 fastapi
 uvicorn
-httpx

services/model-api/training/churn/train.py

@@ -7,6 +7,7 @@
 import argparse
 import os
 import sys
+import requests
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..'))
 
@@ -58,6 +59,24 @@ def main(args):
         print(f"test_roc_auc: {test_metrics['roc_auc']:.4f}")
         print(f'MLflow에 모델 등록 완료: churn-{args.version}')
 
+    github_token = os.getenv("GITHUB_TOKEN")
+    github_owner = os.getenv("GITHUB_REPO_OWNER")
+    github_repo = os.getenv("GITHUB_REPO_NAME")
+
+    if github_token and github_owner and github_repo:
+        response = requests.post(
+            f"https://api.github.com/repos/{github_owner}/{github_repo}/dispatches",
+            headers={
+                "Authorization": f"Bearer {github_token}",
+                "Accept": "application/vnd.github+json",
+            },
+            json={"event_type": "deploy-churn"},
+        )
+        if response.status_code == 204:
+            print("배포 트리거 완료")
+        else:
+            print(f"배포 트리거 실패: {response.status_code}")
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()