diff --git a/.github/workflows/deploy-churn.yaml b/.github/workflows/deploy-churn.yaml new file mode 100644 index 0000000..ba15928 --- /dev/null +++ b/.github/workflows/deploy-churn.yaml @@ -0,0 +1,22 @@ +name: Deploy (churn) + +on: + repository_dispatch: + types: + - deploy-churn + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: kubectl 설정 + run: | + mkdir -p $HOME/.kube + echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config + + - name: model-api 파드 재시작 + run: kubectl rollout restart deployment/model-api -n model-api + + - name: model-api 배포 확인 + run: kubectl rollout status deployment/model-api -n model-api diff --git a/.github/workflows/retrain-churn.yaml b/.github/workflows/retrain-churn.yaml deleted file mode 100644 index 944bcb0..0000000 --- a/.github/workflows/retrain-churn.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Retrain (churn) - -on: - repository_dispatch: - types: - - model-retrain - -jobs: - retrain: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Python 설치 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: 의존성 설치 - run: | - pip install -r services/model-api/requirements.txt - - - name: 트리거된 알림 출력 - run: | - echo "Triggered by alerts: ${{ toJson(github.event.client_payload.alerts) }}" - - - name: 모델 재학습 & MLflow 등록 - working-directory: services/model-api - env: - MLFLOW_TRACKING_URI: https://mlflow.swmlops.site - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ap-northeast-2 - run: | - python training/churn/train.py \ - --mlflow_uri https://mlflow.swmlops.site \ - --data_dir ../data/churn - - - name: model-api 파드 재시작 - run: | - mkdir -p $HOME/.kube - echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config - kubectl rollout restart deployment/model-api -n model-api - kubectl rollout status deployment/model-api -n model-api diff --git a/k8s/alert-bridge/deployment.yaml b/k8s/alert-bridge/deployment.yaml index d8aa97d..07fb293 100644 --- a/k8s/alert-bridge/deployment.yaml +++ b/k8s/alert-bridge/deployment.yaml @@ -19,12 +19,33 @@ spec: ports: - containerPort: 8080 env: - - name: GITHUB_TOKEN + - name: SMTP_HOST valueFrom: secretKeyRef: - name: github-credentials - key: token - - name: GITHUB_REPO_OWNER - value: simGPT - - name: GITHUB_REPO_NAME - value: sw-mlops + name: smtp-credentials + key: host + - name: SMTP_PORT + valueFrom: + secretKeyRef: + name: smtp-credentials + key: port + - name: SMTP_USER + valueFrom: + secretKeyRef: + name: smtp-credentials + key: user + - name: SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: smtp-credentials + key: password + - name: ALERT_MAIL_FROM + valueFrom: + secretKeyRef: + name: smtp-credentials + key: mail-from + - name: ALERT_MAIL_TO + valueFrom: + secretKeyRef: + name: smtp-credentials + key: mail-to diff --git a/services/alert-bridge/main.py b/services/alert-bridge/main.py index 570c987..eea1d9f 100644 --- a/services/alert-bridge/main.py +++ b/services/alert-bridge/main.py @@ -1,36 +1,57 @@ import os -import httpx +import smtplib +import ssl +from datetime import datetime, timezone +from email.mime.text import MIMEText from fastapi import FastAPI, Request app = FastAPI(title="Alert Bridge") -GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") -GITHUB_REPO_OWNER = os.getenv("GITHUB_REPO_OWNER") -GITHUB_REPO_NAME = os.getenv("GITHUB_REPO_NAME") +SMTP_HOST = os.getenv("SMTP_HOST") +SMTP_PORT = int(os.getenv("SMTP_PORT", "465")) +SMTP_USER = os.getenv("SMTP_USER") +SMTP_PASSWORD = os.getenv("SMTP_PASSWORD") +ALERT_MAIL_FROM = os.getenv("ALERT_MAIL_FROM") +ALERT_MAIL_TO = os.getenv("ALERT_MAIL_TO") -# Alertmanager에서 알림을 받아서 GitHub Actions로 전달하는 역할을 하는 api 엔드포인트(Alertmanager가 요청을 보냄) +# 알림 내용을 이메일로 작성하는 함수 +def build_email_body(alerts: list) -> str: + lines = [f"[sw-mlops 알림] {len(alerts)}개의 이상 감지 - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}\n"] + for a in alerts: + labels = a.get("labels", {}) + annotations = a.get("annotations", {}) + lines.append(f"• Alert : {labels.get('alertname', '-')}") + lines.append(f" Severity: {labels.get('severity', '-')}") + lines.append(f" Summary : {annotations.get('summary', '-')}") + lines.append(f" Detail : {annotations.get('description', '-')}\n") + return "\n".join(lines) + +# 이메일 전송하는 함수 +def send_email(subject: str, body: str): + msg = MIMEText(body, "plain", "utf-8") + msg["Subject"] = subject + msg["From"] = ALERT_MAIL_FROM + msg["To"] = ALERT_MAIL_TO + + with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as server: + server.starttls(context=ssl.create_default_context()) + server.login(SMTP_USER, SMTP_PASSWORD) + server.sendmail(ALERT_MAIL_FROM, ALERT_MAIL_TO, msg.as_string()) + + +# 알림 받아서 이메일 전송하는 api @app.post("/alert") async def receive_alert(request: Request): payload = await request.json() - # alertmanager에서 받은 알림 중에서 firing 상태인 알림만 필터링 firing_alerts = [a for a in payload.get("alerts", []) if a["status"] == "firing"] if not firing_alerts: - return {"message": "no firing alerts"} + return {"message": "발생중인 알림이 없습니다."} alert_names = [a["labels"].get("alertname", "") for a in firing_alerts] + subject = f"[sw-mlops 알림] {', '.join(alert_names)}" + body = build_email_body(firing_alerts) # 알림 내용을 이메일 body로 작성 + + send_email(subject, body) # 이메일 전송 - async with httpx.AsyncClient() as client: - response = await client.post( - f"https://api.github.com/repos/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/dispatches", - headers={ - "Authorization": f"Bearer {GITHUB_TOKEN}", - "Accept": "application/vnd.github+json", - }, - json={ - "event_type": "model-retrain", - "client_payload": {"alerts": alert_names}, - }, - ) - - return {"status": response.status_code, "alerts": alert_names} + return {"message": "이메일이 전송되었습니다.", "alerts": alert_names} diff --git a/services/alert-bridge/requirements.txt b/services/alert-bridge/requirements.txt index d23d558..97dc7cd 100644 --- a/services/alert-bridge/requirements.txt +++ b/services/alert-bridge/requirements.txt @@ -1,3 +1,2 @@ fastapi uvicorn -httpx diff --git a/services/model-api/training/churn/train.py b/services/model-api/training/churn/train.py index dfd190e..097737e 100644 --- a/services/model-api/training/churn/train.py +++ b/services/model-api/training/churn/train.py @@ -7,6 +7,7 @@ import argparse import os import sys +import requests sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..')) @@ -58,6 +59,24 @@ def main(args): print(f"test_roc_auc: {test_metrics['roc_auc']:.4f}") print(f'MLflow에 모델 등록 완료: churn-{args.version}') + github_token = os.getenv("GITHUB_TOKEN") + github_owner = os.getenv("GITHUB_REPO_OWNER") + github_repo = os.getenv("GITHUB_REPO_NAME") + + if github_token and github_owner and github_repo: + response = requests.post( + f"https://api.github.com/repos/{github_owner}/{github_repo}/dispatches", + headers={ + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json", + }, + json={"event_type": "deploy-churn"}, + ) + if response.status_code == 204: + print("배포 트리거 완료") + else: + print(f"배포 트리거 실패: {response.status_code}") + if __name__ == '__main__': parser = argparse.ArgumentParser()