🚀 Deploy: 모델 성능 저하에 따른 재학습 파이프라인 구축 및 버전 리펙토링

.github/workflows/cd-production.yaml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - 'services/model-api/**'
+      - 'services/alert-bridge/**'
 
 jobs:
   deploy:
@@ -20,11 +21,17 @@ jobs:
           mkdir -p $HOME/.kube
           echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config
 
-      - name: kubectl apply
+      - name: model-api kubectl apply
         run: kubectl apply -k k8s/model-api/overlays/production/
 
-      - name: 파드 재시작 # (이미지 태그가 계속 latest여도 새 이미지로 배포되도록)
-        run: kubectl rollout restart deployment/model-api -n model-api # 네임스페이스 지정
+      - name: model-api 파드 재시작
+        run: kubectl rollout restart deployment/model-api -n model-api
 
-      - name: 배포 확인
-        run: kubectl rollout status deployment/model-api -n model-api # 네임스페이스 지정
+      - name: model-api 배포 확인
+        run: kubectl rollout status deployment/model-api -n model-api
+
+      - name: alert-bridge 파드 재시작
+        run: kubectl rollout restart deployment/alert-bridge -n monitoring
+
+      - name: alert-bridge 배포 확인
+        run: kubectl rollout status deployment/alert-bridge -n monitoring

.github/workflows/ci-alert-bridge.yaml

@@ -0,0 +1,37 @@
+name: CI (alert-bridge)
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'services/alert-bridge/**'
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - 'services/alert-bridge/**'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker Hub 로그인
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Docker 이미지 빌드 & push
+        uses: docker/build-push-action@v5
+        with:
+          context: services/alert-bridge
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/alert-bridge:latest
+            ${{ secrets.DOCKER_USERNAME }}/alert-bridge:${{ github.sha }}

.github/workflows/retrain-churn.yaml

@@ -0,0 +1,43 @@
+name: Retrain (churn)
+
+on:
+  repository_dispatch:
+    types:
+      - model-retrain
+
+jobs:
+  retrain:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Python 설치
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: 의존성 설치
+        run: |
+          pip install -r services/model-api/requirements.txt
+
+      - name: 트리거된 알림 출력
+        run: |
+          echo "Triggered by alerts: ${{ toJson(github.event.client_payload.alerts) }}"
+
+      - name: 모델 재학습 & MLflow 등록
+        working-directory: services/model-api
+        env:
+          MLFLOW_TRACKING_URI: https://mlflow.swmlops.site
+        run: |
+          python training/churn/train.py \
+            --mlflow_uri https://mlflow.swmlops.site \
+            --data_dir ../data/churn
+
+      - name: model-api 파드 재시작
+        run: |
+          mkdir -p $HOME/.kube
+          echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config
+          kubectl rollout restart deployment/model-api -n model-api
+          kubectl rollout status deployment/model-api -n model-api

.gitignore

@@ -1,5 +1,6 @@
 # 데이터
 data/
+!services/data/churn/
 
 # 모델 가중치 (v1은 예외적으로 커밋할 경우 별도 처리)
 services/model-api/artifacts/*.pth

k8s/alert-bridge/deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alert-bridge
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: alert-bridge
+  template:
+    metadata:
+      labels:
+        app: alert-bridge
+    spec:
+      containers:
+        - name: alert-bridge
+          image: simgpt/alert-bridge:latest
+          ports:
+            - containerPort: 8080
+          env:
+            - name: GITHUB_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: github-credentials
+                  key: token
+            - name: GITHUB_REPO_OWNER
+              value: simGPT
+            - name: GITHUB_REPO_NAME
+              value: sw-mlops

k8s/alert-bridge/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alert-bridge
+  namespace: monitoring
+spec:
+  type: ClusterIP
+  selector:
+    app: alert-bridge
+  ports:
+    - port: 8080
+      targetPort: 8080

k8s/mlflow/deployment.yaml

@@ -41,5 +41,5 @@ spec:
             - --port=5000
             - --default-artifact-root=s3://kube-ml-bucket/mlflow-artifacts # 모델 저장
             - --backend-store-uri=postgresql://mlflow:1234@mlflow-db:5432/mlflow
-            - --allowed-hosts=mlflow.mlflow.svc.cluster.local,localhost,localhost:5000,mlflow.swmlops.site # 풀 DNS, localhost, 도메인 허용
+            - --allowed-hosts=mlflow.mlflow.svc.cluster.local,mlflow.mlflow.svc.cluster.local:5000,localhost,localhost:5000,mlflow.swmlops.site # 풀 DNS, localhost, 도메인 허용
 

k8s/monitoring/alert-rules.yaml

@@ -0,0 +1,60 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: churn-model-alerts
+  namespace: monitoring
+  labels:
+    release: monitoring
+spec:
+  groups:
+    - name: churn-model
+      rules:
+        - alert: ChurnPredictionRatioHigh # 이탈 비율 10분동안 70% 초과
+          expr: |
+            sum(rate(churn_predictions_total{predicted="1"}[10m]))
+            /
+            sum(rate(churn_predictions_total[10m])) > 0.7
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "이탈 예측 비율 이상"
+            description: "이탈 예측 비율이 10분 이상 70%를 초과했습니다."
+
+        - alert: ChurnConfidenceLow # 예측 신뢰도 저하 10분동안 평균 confidence가 0.6 미만
+          expr: |
+            rate(churn_confidence_score_sum[10m])
+            /
+            rate(churn_confidence_score_count[10m]) < 0.6
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "모델 신뢰도 저하"
+            description: "평균 confidence가 10분 이상 0.6 미만입니다."
+
+        - alert: ChurnInferenceLatencyHigh # 추론 지연 시간 5분동안 p99 지연 시간이 0.1초 초과
+          expr: |
+            histogram_quantile(0.99, rate(churn_inference_duration_seconds_bucket[5m])) > 0.1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "추론 지연 시간 이상"
+            description: "추론 p99 지연 시간이 5분 이상 0.1초를 초과했습니다."
+
+        - alert: ChurnFeatureDrift # 피처 평균이 24시간 전 대비 50% 이상 변화
+          expr: |
+            abs(
+              avg_over_time(churn_feature_mean[1h])
+              -
+              avg_over_time(churn_feature_mean[1h] offset 24h)
+            )
+            /
+            avg_over_time(churn_feature_mean[1h] offset 24h) > 0.5
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "입력 피처 분포 이상"
+            description: "피처 평균이 24시간 전 대비 50% 이상 변화했습니다."

k8s/monitoring/values.yaml

@@ -6,6 +6,26 @@ prometheus:
           - targets:
               - model-api.model-api.svc.cluster.local:8000 # 다른 네임스페이스에 있는 model-api의 클러스터 내부 주소와 포트
 
+alertmanager:
+  config:
+    global:
+      resolve_timeout: 5m # 알림이 해결된 것으로 간주하기 전에 기다리는 시간 5분
+    route:
+      group_by: ["alertname"] # 같은 alertname을 가진 알림들을 그룹화
+      group_wait: 30s # 첫 번째 알림이 발생한 후 그룹화된 알림을 전송하기 전에 기다리는 시간 30초
+      group_interval: 5m # 그룹화된 알림을 전송한 후 다음 알림을 전송하기 전에 기다리는 시간 5분
+      repeat_interval: 12h # 같은 알림이 계속 발생할 때 알림을 반복해서 전송하는 간격 12시간
+      receiver: alert-bridge # alert-bridge로 알림 전송
+      routes:
+        - matchers:
+            - alertname =~ "ChurnPrediction.*|ChurnConfidence.*|ChurnInference.*|ChurnFeature.*"
+          receiver: alert-bridge
+    receivers:
+      - name: alert-bridge
+        webhook_configs:
+          - url: http://alert-bridge.monitoring.svc.cluster.local:8080/alert # alert-bridge의 클러스터 내부 주소와 포트
+            send_resolved: false # 알림이 해결되지 않은 경우에만 알림 전송
+
 grafana:
   ingress:
     enabled: true

services/alert-bridge/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY main.py .
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]

services/alert-bridge/main.py

@@ -0,0 +1,36 @@
+import os
+import httpx
+from fastapi import FastAPI, Request
+
+app = FastAPI(title="Alert Bridge")
+
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+GITHUB_REPO_OWNER = os.getenv("GITHUB_REPO_OWNER")
+GITHUB_REPO_NAME = os.getenv("GITHUB_REPO_NAME")
+
+# Alertmanager에서 알림을 받아서 GitHub Actions로 전달하는 역할을 하는 api 엔드포인트(Alertmanager가 요청을 보냄)
+@app.post("/alert")
+async def receive_alert(request: Request):
+    payload = await request.json()
+
+    # alertmanager에서 받은 알림 중에서 firing 상태인 알림만 필터링
+    firing_alerts = [a for a in payload.get("alerts", []) if a["status"] == "firing"]
+    if not firing_alerts:
+        return {"message": "no firing alerts"}
+
+    alert_names = [a["labels"].get("alertname", "") for a in firing_alerts]
+
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            f"https://api.github.com/repos/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/dispatches",
+            headers={
+                "Authorization": f"Bearer {GITHUB_TOKEN}",
+                "Accept": "application/vnd.github+json",
+            },
+            json={
+                "event_type": "model-retrain",
+                "client_payload": {"alerts": alert_names},
+            },
+        )
+
+    return {"status": response.status_code, "alerts": alert_names}

services/alert-bridge/requirements.txt

@@ -0,0 +1,3 @@
+fastapi
+uvicorn
+httpx

services/model-api/app/models/loader.py

@@ -6,21 +6,36 @@
 _model_cache: dict = {}
 
 # mlflow에서 mnist 모델 로드하는 함수
-def load_model(model_name: str, version: str):
-    cache_key = f"{model_name}_{version}"
+def load_model(model_name: str):
+    cache_key = model_name
     if cache_key in _model_cache:
         return _model_cache[cache_key]
 
     mlflow_uri = os.getenv("MLFLOW_TRACKING_URI", "http://mlflow:5000") # 환경변수에서 mlflow tracking uri 가져오기, 없으면 기본값으로 http://mlflow:5000 사용
     mlflow.set_tracking_uri(mlflow_uri)
 
-    model_uri = f"models:/{model_name}-{version}/latest"
+    model_uri = f"models:/{model_name}/latest"
     model = mlflow.pytorch.load_model(model_uri)
     model.eval()
 
     _model_cache[cache_key] = model
     return model
 
+# mlflow에서 고객 이탈 예측 모델 로드하는 함수
+def load_churn_model(model_name: str):
+    cache_key = f"{model_name}_sklearn"
+    if cache_key in _model_cache:
+        return _model_cache[cache_key]
+
+    mlflow_uri = os.getenv("MLFLOW_TRACKING_URI", "http://mlflow:5000") # 환경변수에서 mlflow tracking uri 가져오기, 없으면 기본값으로 http://mlflow:5000 사용
+    mlflow.set_tracking_uri(mlflow_uri)
+
+    model_uri = f"models:/{model_name}/latest"
+    model = mlflow.sklearn.load_model(model_uri)
+
+    _model_cache[cache_key] = model
+    return model
+
 # mlflow에서 고객 이탈 예측 모델 로드하는 함수
 def load_churn_model(model_name: str, version: str):
     cache_key = f"{model_name}_{version}_sklearn"