✨ Feat: 모델 성능 저하 자동 감지 및 재학습 파이프라인 구축

.github/workflows/cd-production.yaml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - 'services/model-api/**'
+      - 'services/alert-bridge/**'
 
 jobs:
   deploy:
@@ -20,11 +21,17 @@ jobs:
           mkdir -p $HOME/.kube
           echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config
 
-      - name: kubectl apply
+      - name: model-api kubectl apply
         run: kubectl apply -k k8s/model-api/overlays/production/
 
-      - name: 파드 재시작 # (이미지 태그가 계속 latest여도 새 이미지로 배포되도록)
-        run: kubectl rollout restart deployment/model-api -n model-api # 네임스페이스 지정
+      - name: model-api 파드 재시작
+        run: kubectl rollout restart deployment/model-api -n model-api
 
-      - name: 배포 확인
-        run: kubectl rollout status deployment/model-api -n model-api # 네임스페이스 지정
+      - name: model-api 배포 확인
+        run: kubectl rollout status deployment/model-api -n model-api
+
+      - name: alert-bridge 파드 재시작
+        run: kubectl rollout restart deployment/alert-bridge -n monitoring
+
+      - name: alert-bridge 배포 확인
+        run: kubectl rollout status deployment/alert-bridge -n monitoring

.github/workflows/ci-alert-bridge.yaml

@@ -0,0 +1,37 @@
+name: CI (alert-bridge)
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'services/alert-bridge/**'
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - 'services/alert-bridge/**'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker Hub 로그인
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Docker 이미지 빌드 & push
+        uses: docker/build-push-action@v5
+        with:
+          context: services/alert-bridge
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/alert-bridge:latest
+            ${{ secrets.DOCKER_USERNAME }}/alert-bridge:${{ github.sha }}

.github/workflows/retrain-churn.yaml

@@ -0,0 +1,43 @@
+name: Retrain (churn)
+
+on:
+  repository_dispatch:
+    types:
+      - model-retrain
+
+jobs:
+  retrain:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Python 설치
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: 의존성 설치
+        run: |
+          pip install -r services/model-api/requirements.txt
+
+      - name: 트리거된 알림 출력
+        run: |
+          echo "Triggered by alerts: ${{ toJson(github.event.client_payload.alerts) }}"
+
+      - name: 모델 재학습 & MLflow 등록
+        working-directory: services/model-api
+        env:
+          MLFLOW_TRACKING_URI: https://mlflow.swmlops.site
+        run: |
+          python training/churn/train.py \
+            --mlflow_uri https://mlflow.swmlops.site \
+            --data_dir ../data/churn
+
+      - name: model-api 파드 재시작
+        run: |
+          mkdir -p $HOME/.kube
+          echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config
+          kubectl rollout restart deployment/model-api -n model-api
+          kubectl rollout status deployment/model-api -n model-api

.gitignore

@@ -1,5 +1,6 @@
 # 데이터
 data/
+!services/data/churn/
 
 # 모델 가중치 (v1은 예외적으로 커밋할 경우 별도 처리)
 services/model-api/artifacts/*.pth

k8s/alert-bridge/deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alert-bridge
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: alert-bridge
+  template:
+    metadata:
+      labels:
+        app: alert-bridge
+    spec:
+      containers:
+        - name: alert-bridge
+          image: simgpt/alert-bridge:latest
+          ports:
+            - containerPort: 8080
+          env:
+            - name: GITHUB_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: github-credentials
+                  key: token
+            - name: GITHUB_REPO_OWNER
+              value: simGPT
+            - name: GITHUB_REPO_NAME
+              value: sw-mlops

k8s/alert-bridge/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alert-bridge
+  namespace: monitoring
+spec:
+  type: ClusterIP
+  selector:
+    app: alert-bridge
+  ports:
+    - port: 8080
+      targetPort: 8080

k8s/monitoring/alert-rules.yaml

@@ -0,0 +1,60 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: churn-model-alerts
+  namespace: monitoring
+  labels:
+    release: monitoring
+spec:
+  groups:
+    - name: churn-model
+      rules:
+        - alert: ChurnPredictionRatioHigh # 이탈 비율 10분동안 70% 초과
+          expr: |
+            sum(rate(churn_predictions_total{predicted="1"}[10m]))
+            /
+            sum(rate(churn_predictions_total[10m])) > 0.7
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "이탈 예측 비율 이상"
+            description: "이탈 예측 비율이 10분 이상 70%를 초과했습니다."
+
+        - alert: ChurnConfidenceLow # 예측 신뢰도 저하 10분동안 평균 confidence가 0.6 미만
+          expr: |
+            rate(churn_confidence_score_sum[10m])
+            /
+            rate(churn_confidence_score_count[10m]) < 0.6
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "모델 신뢰도 저하"
+            description: "평균 confidence가 10분 이상 0.6 미만입니다."
+
+        - alert: ChurnInferenceLatencyHigh # 추론 지연 시간 5분동안 p99 지연 시간이 0.1초 초과
+          expr: |
+            histogram_quantile(0.99, rate(churn_inference_duration_seconds_bucket[5m])) > 0.1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "추론 지연 시간 이상"
+            description: "추론 p99 지연 시간이 5분 이상 0.1초를 초과했습니다."
+
+        - alert: ChurnFeatureDrift # 피처 평균이 24시간 전 대비 50% 이상 변화
+          expr: |
+            abs(
+              avg_over_time(churn_feature_mean[1h])
+              -
+              avg_over_time(churn_feature_mean[1h] offset 24h)
+            )
+            /
+            avg_over_time(churn_feature_mean[1h] offset 24h) > 0.5
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "입력 피처 분포 이상"
+            description: "피처 평균이 24시간 전 대비 50% 이상 변화했습니다."

k8s/monitoring/values.yaml

@@ -6,6 +6,26 @@ prometheus:
           - targets:
               - model-api.model-api.svc.cluster.local:8000 # 다른 네임스페이스에 있는 model-api의 클러스터 내부 주소와 포트
 
+alertmanager:
+  config:
+    global:
+      resolve_timeout: 5m # 알림이 해결된 것으로 간주하기 전에 기다리는 시간 5분
+    route:
+      group_by: ["alertname"] # 같은 alertname을 가진 알림들을 그룹화
+      group_wait: 30s # 첫 번째 알림이 발생한 후 그룹화된 알림을 전송하기 전에 기다리는 시간 30초
+      group_interval: 5m # 그룹화된 알림을 전송한 후 다음 알림을 전송하기 전에 기다리는 시간 5분
+      repeat_interval: 12h # 같은 알림이 계속 발생할 때 알림을 반복해서 전송하는 간격 12시간
+      receiver: alert-bridge # alert-bridge로 알림 전송
+      routes:
+        - matchers:
+            - alertname =~ "ChurnPrediction.*|ChurnConfidence.*|ChurnInference.*|ChurnFeature.*"
+          receiver: alert-bridge
+    receivers:
+      - name: alert-bridge
+        webhook_configs:
+          - url: http://alert-bridge.monitoring.svc.cluster.local:8080/alert # alert-bridge의 클러스터 내부 주소와 포트
+            send_resolved: false # 알림이 해결되지 않은 경우에만 알림 전송
+
 grafana:
   ingress:
     enabled: true

services/alert-bridge/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY main.py .
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]

services/alert-bridge/main.py

@@ -0,0 +1,36 @@
+import os
+import httpx
+from fastapi import FastAPI, Request
+
+app = FastAPI(title="Alert Bridge")
+
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+GITHUB_REPO_OWNER = os.getenv("GITHUB_REPO_OWNER")
+GITHUB_REPO_NAME = os.getenv("GITHUB_REPO_NAME")
+
+# Alertmanager에서 알림을 받아서 GitHub Actions로 전달하는 역할을 하는 api 엔드포인트(Alertmanager가 요청을 보냄)
+@app.post("/alert")
+async def receive_alert(request: Request):
+    payload = await request.json()
+
+    # alertmanager에서 받은 알림 중에서 firing 상태인 알림만 필터링
+    firing_alerts = [a for a in payload.get("alerts", []) if a["status"] == "firing"]
+    if not firing_alerts:
+        return {"message": "no firing alerts"}
+
+    alert_names = [a["labels"].get("alertname", "") for a in firing_alerts]
+
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            f"https://api.github.com/repos/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/dispatches",
+            headers={
+                "Authorization": f"Bearer {GITHUB_TOKEN}",
+                "Accept": "application/vnd.github+json",
+            },
+            json={
+                "event_type": "model-retrain",
+                "client_payload": {"alerts": alert_names},
+            },
+        )
+
+    return {"status": response.status_code, "alerts": alert_names}

services/alert-bridge/requirements.txt

@@ -0,0 +1,3 @@
+fastapi
+uvicorn
+httpx

services/model-api/app/services/churn_service.py

@@ -3,7 +3,7 @@
 from datetime import datetime, timezone
 
 from app.models.loader import load_churn_model
-from prometheus_client import Counter, Histogram
+from prometheus_client import Counter, Gauge, Histogram
 
 MODEL_NAME = "churn"
 
@@ -18,16 +18,31 @@
     'cart_abandonment_rate',
 ]
 
-prediction_counter = Counter(
+# Prometheus 메트릭 정의
+# 총 예측 횟수, 예측된 클래스별로 레이블(predicted) 추가
+prediction_counter = Counter( # Counter는 총 횟수를 세는 메트릭
     "churn_predictions_total",
     "이탈 예측 횟수",
     ["predicted"],
 )
-confidence_histogram = Histogram(
+# 예측 신뢰도 분포
+confidence_histogram = Histogram( # Histogram은 값의 분포를 나타내는 메트릭
     "churn_confidence_score",
     "이탈 예측 신뢰도 분포",
     buckets=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
 )
+# 입력 피처의 평균값
+feature_mean_gauge = Gauge( # Gauge는 현재 값을 나타내는 메트릭
+    "churn_feature_mean",
+    "입력 피처 평균값",
+    ["feature"],
+)
+# 추론 지연 시간
+inference_histogram = Histogram(
+    "churn_inference_duration_seconds",
+    "추론 지연 시간",
+    buckets=[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0],
+)
 
 
 def predict(data: dict) -> dict:
@@ -39,14 +54,19 @@ def predict(data: dict) -> dict:
 
     x = [[data[f] for f in FEATURES]]
 
+    for feature in FEATURES:
+        feature_mean_gauge.labels(feature=feature).set(data[feature])
+
     start = time.time()
     predicted = int(model.predict(x)[0])
     proba = model.predict_proba(x)[0]
     confidence = float(proba[predicted])
-    time_ms = round((time.time() - start) * 1000, 3)
+    duration = time.time() - start
+    time_ms = round(duration * 1000, 3)
 
     prediction_counter.labels(predicted=str(predicted)).inc()
     confidence_histogram.observe(confidence)
+    inference_histogram.observe(duration)
 
     return {
         "result": {