From 88db24e464e7f7825661f6a4db9b8f2c469b1f1a Mon Sep 17 00:00:00 2001 From: simGPT Date: Fri, 5 Jun 2026 20:41:52 +0900 Subject: [PATCH 1/9] =?UTF-8?q?:wrench:Settings:=20smtp=20=EA=B4=80?= =?UTF-8?q?=EB=A0=A8=20=ED=99=98=EA=B2=BD=EB=B3=80=EC=88=98=20secret?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- k8s/alert-bridge/deployment.yaml | 35 +++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/k8s/alert-bridge/deployment.yaml b/k8s/alert-bridge/deployment.yaml index d8aa97d..07fb293 100644 --- a/k8s/alert-bridge/deployment.yaml +++ b/k8s/alert-bridge/deployment.yaml @@ -19,12 +19,33 @@ spec: ports: - containerPort: 8080 env: - - name: GITHUB_TOKEN + - name: SMTP_HOST valueFrom: secretKeyRef: - name: github-credentials - key: token - - name: GITHUB_REPO_OWNER - value: simGPT - - name: GITHUB_REPO_NAME - value: sw-mlops + name: smtp-credentials + key: host + - name: SMTP_PORT + valueFrom: + secretKeyRef: + name: smtp-credentials + key: port + - name: SMTP_USER + valueFrom: + secretKeyRef: + name: smtp-credentials + key: user + - name: SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: smtp-credentials + key: password + - name: ALERT_MAIL_FROM + valueFrom: + secretKeyRef: + name: smtp-credentials + key: mail-from + - name: ALERT_MAIL_TO + valueFrom: + secretKeyRef: + name: smtp-credentials + key: mail-to From 1de2e3de0adeff2cd75f2050ec6dc06058bde9c4 Mon Sep 17 00:00:00 2001 From: simGPT Date: Fri, 5 Jun 2026 20:43:20 +0900 Subject: [PATCH 2/9] =?UTF-8?q?:recycle:Refactor:=20webhook=20=EC=9A=94?= =?UTF-8?q?=EC=B2=AD=20=EC=8B=9C=20=EC=9D=B4=EB=A9=94=EC=9D=BC=20=EB=B0=9C?= =?UTF-8?q?=EC=86=A1=EC=9A=A9=20api=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/alert-bridge/main.py | 63 +++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/services/alert-bridge/main.py b/services/alert-bridge/main.py index 570c987..7e19e64 100644 --- a/services/alert-bridge/main.py +++ b/services/alert-bridge/main.py @@ -1,36 +1,57 @@ import os -import httpx +import smtplib +import ssl +from datetime import datetime, timezone +from email.mime.text import MIMEText from fastapi import FastAPI, Request app = FastAPI(title="Alert Bridge") -GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") -GITHUB_REPO_OWNER = os.getenv("GITHUB_REPO_OWNER") -GITHUB_REPO_NAME = os.getenv("GITHUB_REPO_NAME") +SMTP_HOST = os.getenv("SMTP_HOST") +SMTP_PORT = int(os.getenv("SMTP_PORT", "465")) +SMTP_USER = os.getenv("SMTP_USER") +SMTP_PASSWORD = os.getenv("SMTP_PASSWORD") +ALERT_MAIL_FROM = os.getenv("ALERT_MAIL_FROM") +ALERT_MAIL_TO = os.getenv("ALERT_MAIL_TO") -# Alertmanager에서 알림을 받아서 GitHub Actions로 전달하는 역할을 하는 api 엔드포인트(Alertmanager가 요청을 보냄) +# 알림 내용을 이메일로 작성하는 함수 +def build_email_body(alerts: list) -> str: + lines = [f"[sw-mlops 알림] {len(alerts)}개의 이상 감지 - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}\n"] + for a in alerts: + labels = a.get("labels", {}) + annotations = a.get("annotations", {}) + lines.append(f"• Alert : {labels.get('alertname', '-')}") + lines.append(f" Severity: {labels.get('severity', '-')}") + lines.append(f" Summary : {annotations.get('summary', '-')}") + lines.append(f" Detail : {annotations.get('description', '-')}\n") + return "\n".join(lines) + +# 이메일 전송하는 함수 +def send_email(subject: str, body: str): + msg = MIMEText(body, "plain", "utf-8") + msg["Subject"] = subject + msg["From"] = ALERT_MAIL_FROM + msg["To"] = ALERT_MAIL_TO + + context = ssl.create_default_context() + with smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT, context=context) as server: + server.login(SMTP_USER, SMTP_PASSWORD) + server.sendmail(ALERT_MAIL_FROM, ALERT_MAIL_TO, msg.as_string()) + + +# 알림 받아서 이메일 전송하는 api @app.post("/alert") async def receive_alert(request: Request): payload = await request.json() - # alertmanager에서 받은 알림 중에서 firing 상태인 알림만 필터링 firing_alerts = [a for a in payload.get("alerts", []) if a["status"] == "firing"] if not firing_alerts: - return {"message": "no firing alerts"} + return {"message": "발생중인 알림이 없습니다."} alert_names = [a["labels"].get("alertname", "") for a in firing_alerts] + subject = f"[sw-mlops 알림] {', '.join(alert_names)}" + body = build_email_body(firing_alerts) # 알림 내용을 이메일 body로 작성 + + send_email(subject, body) # 이메일 전송 - async with httpx.AsyncClient() as client: - response = await client.post( - f"https://api.github.com/repos/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/dispatches", - headers={ - "Authorization": f"Bearer {GITHUB_TOKEN}", - "Accept": "application/vnd.github+json", - }, - json={ - "event_type": "model-retrain", - "client_payload": {"alerts": alert_names}, - }, - ) - - return {"status": response.status_code, "alerts": alert_names} + return {"message": "이메일이 전송되었습니다.", "alerts": alert_names} From 9720a75bf192674a9c210773eceaa101ed2317b9 Mon Sep 17 00:00:00 2001 From: simGPT Date: Fri, 5 Jun 2026 20:45:26 +0900 Subject: [PATCH 3/9] =?UTF-8?q?:heavy=5Fminus=5Fsign:Dependency:=20httpx?= =?UTF-8?q?=20=EC=9D=98=EC=A1=B4=EC=84=B1=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/alert-bridge/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/services/alert-bridge/requirements.txt b/services/alert-bridge/requirements.txt index d23d558..97dc7cd 100644 --- a/services/alert-bridge/requirements.txt +++ b/services/alert-bridge/requirements.txt @@ -1,3 +1,2 @@ fastapi uvicorn -httpx From fee97ee60e1432b63154015ba93a21743e32848f Mon Sep 17 00:00:00 2001 From: simGPT Date: Fri, 5 Jun 2026 20:48:15 +0900 Subject: [PATCH 4/9] =?UTF-8?q?:fire:Remove:=20=EC=9E=90=EB=8F=99=20?= =?UTF-8?q?=EC=9E=AC=ED=95=99=EC=8A=B5=20=EC=84=A4=EC=A0=95=20=ED=8C=8C?= =?UTF-8?q?=EC=9D=BC=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/retrain-churn.yaml | 46 ---------------------------- 1 file changed, 46 deletions(-) delete mode 100644 .github/workflows/retrain-churn.yaml diff --git a/.github/workflows/retrain-churn.yaml b/.github/workflows/retrain-churn.yaml deleted file mode 100644 index 944bcb0..0000000 --- a/.github/workflows/retrain-churn.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Retrain (churn) - -on: - repository_dispatch: - types: - - model-retrain - -jobs: - retrain: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Python 설치 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: 의존성 설치 - run: | - pip install -r services/model-api/requirements.txt - - - name: 트리거된 알림 출력 - run: | - echo "Triggered by alerts: ${{ toJson(github.event.client_payload.alerts) }}" - - - name: 모델 재학습 & MLflow 등록 - working-directory: services/model-api - env: - MLFLOW_TRACKING_URI: https://mlflow.swmlops.site - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ap-northeast-2 - run: | - python training/churn/train.py \ - --mlflow_uri https://mlflow.swmlops.site \ - --data_dir ../data/churn - - - name: model-api 파드 재시작 - run: | - mkdir -p $HOME/.kube - echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config - kubectl rollout restart deployment/model-api -n model-api - kubectl rollout status deployment/model-api -n model-api From 0f96ee2c19dbb9a6fe797164ac75cbe8f12b139d Mon Sep 17 00:00:00 2001 From: simGPT Date: Fri, 5 Jun 2026 21:28:31 +0900 Subject: [PATCH 5/9] =?UTF-8?q?:wrench:Settings:=20SMTP=20SSL=20->=20TLS?= =?UTF-8?q?=20=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/alert-bridge/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/alert-bridge/main.py b/services/alert-bridge/main.py index 7e19e64..eea1d9f 100644 --- a/services/alert-bridge/main.py +++ b/services/alert-bridge/main.py @@ -33,8 +33,8 @@ def send_email(subject: str, body: str): msg["From"] = ALERT_MAIL_FROM msg["To"] = ALERT_MAIL_TO - context = ssl.create_default_context() - with smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT, context=context) as server: + with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as server: + server.starttls(context=ssl.create_default_context()) server.login(SMTP_USER, SMTP_PASSWORD) server.sendmail(ALERT_MAIL_FROM, ALERT_MAIL_TO, msg.as_string()) From 57d85fb340deaf6009e31aa2d0079b36189e9dae Mon Sep 17 00:00:00 2001 From: simGPT Date: Sat, 6 Jun 2026 01:24:35 +0900 Subject: [PATCH 6/9] =?UTF-8?q?:sparkles:Feat:=20=ED=95=99=EC=8A=B5=20?= =?UTF-8?q?=ED=9B=84=20=EC=9E=90=EB=8F=99=20=EB=B0=B0=ED=8F=AC=20=EB=A1=9C?= =?UTF-8?q?=EC=A7=81=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/model-api/training/churn/train.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/services/model-api/training/churn/train.py b/services/model-api/training/churn/train.py index dfd190e..06e7be8 100644 --- a/services/model-api/training/churn/train.py +++ b/services/model-api/training/churn/train.py @@ -7,6 +7,7 @@ import argparse import os import sys +import requests sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..')) @@ -58,6 +59,24 @@ def main(args): print(f"test_roc_auc: {test_metrics['roc_auc']:.4f}") print(f'MLflow에 모델 등록 완료: churn-{args.version}') + github_token = os.getenv("GITHUB_TOKEN") + github_owner = os.getenv("GITHUB_REPO_OWNER") + github_repo = os.getenv("GITHUB_REPO_NAME") + + if github_token and github_owner and github_repo: + response = requests.post( + f"https://api.github.com/repos/{github_owner}/{github_repo}/dispatches", + headers={ + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json", + }, + json={"event_type": "model-deploy"}, + ) + if response.status_code == 204: + print("배포 트리거 완료") + else: + print(f"배포 트리거 실패: {response.status_code}") + if __name__ == '__main__': parser = argparse.ArgumentParser() From 4e9fbf88fc0de6092968d838b695779bbe3ff1d8 Mon Sep 17 00:00:00 2001 From: simGPT Date: Sat, 6 Jun 2026 01:31:42 +0900 Subject: [PATCH 7/9] =?UTF-8?q?:card=5Ffile=5Fbox:Comment:=20=EC=A3=BC?= =?UTF-8?q?=EC=84=9D=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/model-api/training/churn/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/model-api/training/churn/train.py b/services/model-api/training/churn/train.py index 06e7be8..c11bf81 100644 --- a/services/model-api/training/churn/train.py +++ b/services/model-api/training/churn/train.py @@ -70,7 +70,7 @@ def main(args): "Authorization": f"Bearer {github_token}", "Accept": "application/vnd.github+json", }, - json={"event_type": "model-deploy"}, + json={"event_type": "model-deploy"}, # deploy-churn 이벤트 트리거 ) if response.status_code == 204: print("배포 트리거 완료") From abea28be3f5dcee9f7d15eaae69110c359f1d8ee Mon Sep 17 00:00:00 2001 From: simGPT Date: Sat, 6 Jun 2026 01:33:06 +0900 Subject: [PATCH 8/9] =?UTF-8?q?:wrench:Settings:=20churn=20=EB=AA=A8?= =?UTF-8?q?=EB=8D=B8=20=EC=9A=B4=EC=98=81=20=EB=B0=B0=ED=8F=AC=20=EC=9B=8C?= =?UTF-8?q?=ED=81=AC=ED=94=8C=EB=A1=9C=EC=9A=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy-churn.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/deploy-churn.yaml diff --git a/.github/workflows/deploy-churn.yaml b/.github/workflows/deploy-churn.yaml new file mode 100644 index 0000000..64c35af --- /dev/null +++ b/.github/workflows/deploy-churn.yaml @@ -0,0 +1,22 @@ +name: Deploy (churn) + +on: + repository_dispatch: + types: + - model-deploy + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: kubectl 설정 + run: | + mkdir -p $HOME/.kube + echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config + + - name: model-api 파드 재시작 + run: kubectl rollout restart deployment/model-api -n model-api + + - name: model-api 배포 확인 + run: kubectl rollout status deployment/model-api -n model-api From 309ba11dfc60c4735954924b62d3188207dde326 Mon Sep 17 00:00:00 2001 From: simGPT Date: Sat, 6 Jun 2026 11:30:04 +0900 Subject: [PATCH 9/9] =?UTF-8?q?:wrench:Settings:=20=EB=B0=B0=ED=8F=AC=20?= =?UTF-8?q?=EC=9D=B4=EB=B2=A4=ED=8A=B8=20=EB=AA=85=20=EB=B3=80=EA=B2=BD(mo?= =?UTF-8?q?del-deploy=20->=20deploy-churn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy-churn.yaml | 2 +- services/model-api/training/churn/train.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-churn.yaml b/.github/workflows/deploy-churn.yaml index 64c35af..ba15928 100644 --- a/.github/workflows/deploy-churn.yaml +++ b/.github/workflows/deploy-churn.yaml @@ -3,7 +3,7 @@ name: Deploy (churn) on: repository_dispatch: types: - - model-deploy + - deploy-churn jobs: deploy: diff --git a/services/model-api/training/churn/train.py b/services/model-api/training/churn/train.py index c11bf81..097737e 100644 --- a/services/model-api/training/churn/train.py +++ b/services/model-api/training/churn/train.py @@ -70,7 +70,7 @@ def main(args): "Authorization": f"Bearer {github_token}", "Accept": "application/vnd.github+json", }, - json={"event_type": "model-deploy"}, # deploy-churn 이벤트 트리거 + json={"event_type": "deploy-churn"}, ) if response.status_code == 204: print("배포 트리거 완료")