diff --git a/.github/workflows/cd-production.yaml b/.github/workflows/cd-production.yaml index 25749fc..1b19550 100644 --- a/.github/workflows/cd-production.yaml +++ b/.github/workflows/cd-production.yaml @@ -6,6 +6,7 @@ on: - main paths: - 'services/model-api/**' + - 'services/alert-bridge/**' jobs: deploy: @@ -20,11 +21,17 @@ jobs: mkdir -p $HOME/.kube echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config - - name: kubectl apply + - name: model-api kubectl apply run: kubectl apply -k k8s/model-api/overlays/production/ - - name: 파드 재시작 # (이미지 태그가 계속 latest여도 새 이미지로 배포되도록) - run: kubectl rollout restart deployment/model-api -n model-api # 네임스페이스 지정 + - name: model-api 파드 재시작 + run: kubectl rollout restart deployment/model-api -n model-api - - name: 배포 확인 - run: kubectl rollout status deployment/model-api -n model-api # 네임스페이스 지정 + - name: model-api 배포 확인 + run: kubectl rollout status deployment/model-api -n model-api + + - name: alert-bridge 파드 재시작 + run: kubectl rollout restart deployment/alert-bridge -n monitoring + + - name: alert-bridge 배포 확인 + run: kubectl rollout status deployment/alert-bridge -n monitoring diff --git a/.github/workflows/ci-alert-bridge.yaml b/.github/workflows/ci-alert-bridge.yaml new file mode 100644 index 0000000..4c84a0a --- /dev/null +++ b/.github/workflows/ci-alert-bridge.yaml @@ -0,0 +1,37 @@ +name: CI (alert-bridge) + +on: + push: + branches: + - develop + paths: + - 'services/alert-bridge/**' + pull_request: + branches: + - main + - develop + paths: + - 'services/alert-bridge/**' + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Docker Hub 로그인 + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Docker 이미지 빌드 & push + uses: docker/build-push-action@v5 + with: + context: services/alert-bridge + push: true + tags: | + ${{ secrets.DOCKER_USERNAME }}/alert-bridge:latest + ${{ secrets.DOCKER_USERNAME }}/alert-bridge:${{ github.sha }} diff --git a/.github/workflows/retrain-churn.yaml b/.github/workflows/retrain-churn.yaml new file mode 100644 index 0000000..91eb520 --- /dev/null +++ b/.github/workflows/retrain-churn.yaml @@ -0,0 +1,43 @@ +name: Retrain (churn) + +on: + repository_dispatch: + types: + - model-retrain + +jobs: + retrain: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Python 설치 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: 의존성 설치 + run: | + pip install -r services/model-api/requirements.txt + + - name: 트리거된 알림 출력 + run: | + echo "Triggered by alerts: ${{ toJson(github.event.client_payload.alerts) }}" + + - name: 모델 재학습 & MLflow 등록 + working-directory: services/model-api + env: + MLFLOW_TRACKING_URI: https://mlflow.swmlops.site + run: | + python training/churn/train.py \ + --mlflow_uri https://mlflow.swmlops.site \ + --data_dir ../data/churn + + - name: model-api 파드 재시작 + run: | + mkdir -p $HOME/.kube + echo "${{ secrets.KUBECONFIG_PRODUCTION }}" | base64 -d > $HOME/.kube/config + kubectl rollout restart deployment/model-api -n model-api + kubectl rollout status deployment/model-api -n model-api diff --git a/.gitignore b/.gitignore index 717e219..c84852f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # 데이터 data/ +!services/data/churn/ # 모델 가중치 (v1은 예외적으로 커밋할 경우 별도 처리) services/model-api/artifacts/*.pth diff --git a/k8s/alert-bridge/deployment.yaml b/k8s/alert-bridge/deployment.yaml new file mode 100644 index 0000000..d8aa97d --- /dev/null +++ b/k8s/alert-bridge/deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alert-bridge + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: alert-bridge + template: + metadata: + labels: + app: alert-bridge + spec: + containers: + - name: alert-bridge + image: simgpt/alert-bridge:latest + ports: + - containerPort: 8080 + env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: github-credentials + key: token + - name: GITHUB_REPO_OWNER + value: simGPT + - name: GITHUB_REPO_NAME + value: sw-mlops diff --git a/k8s/alert-bridge/service.yaml b/k8s/alert-bridge/service.yaml new file mode 100644 index 0000000..bc85f35 --- /dev/null +++ b/k8s/alert-bridge/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: alert-bridge + namespace: monitoring +spec: + type: ClusterIP + selector: + app: alert-bridge + ports: + - port: 8080 + targetPort: 8080 diff --git a/k8s/monitoring/alert-rules.yaml b/k8s/monitoring/alert-rules.yaml new file mode 100644 index 0000000..44649c2 --- /dev/null +++ b/k8s/monitoring/alert-rules.yaml @@ -0,0 +1,60 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: churn-model-alerts + namespace: monitoring + labels: + release: monitoring +spec: + groups: + - name: churn-model + rules: + - alert: ChurnPredictionRatioHigh # 이탈 비율 10분동안 70% 초과 + expr: | + sum(rate(churn_predictions_total{predicted="1"}[10m])) + / + sum(rate(churn_predictions_total[10m])) > 0.7 + for: 10m + labels: + severity: warning + annotations: + summary: "이탈 예측 비율 이상" + description: "이탈 예측 비율이 10분 이상 70%를 초과했습니다." + + - alert: ChurnConfidenceLow # 예측 신뢰도 저하 10분동안 평균 confidence가 0.6 미만 + expr: | + rate(churn_confidence_score_sum[10m]) + / + rate(churn_confidence_score_count[10m]) < 0.6 + for: 10m + labels: + severity: warning + annotations: + summary: "모델 신뢰도 저하" + description: "평균 confidence가 10분 이상 0.6 미만입니다." + + - alert: ChurnInferenceLatencyHigh # 추론 지연 시간 5분동안 p99 지연 시간이 0.1초 초과 + expr: | + histogram_quantile(0.99, rate(churn_inference_duration_seconds_bucket[5m])) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "추론 지연 시간 이상" + description: "추론 p99 지연 시간이 5분 이상 0.1초를 초과했습니다." + + - alert: ChurnFeatureDrift # 피처 평균이 24시간 전 대비 50% 이상 변화 + expr: | + abs( + avg_over_time(churn_feature_mean[1h]) + - + avg_over_time(churn_feature_mean[1h] offset 24h) + ) + / + avg_over_time(churn_feature_mean[1h] offset 24h) > 0.5 + for: 10m + labels: + severity: warning + annotations: + summary: "입력 피처 분포 이상" + description: "피처 평균이 24시간 전 대비 50% 이상 변화했습니다." diff --git a/k8s/monitoring/values.yaml b/k8s/monitoring/values.yaml index e6d476f..66a3959 100644 --- a/k8s/monitoring/values.yaml +++ b/k8s/monitoring/values.yaml @@ -6,6 +6,26 @@ prometheus: - targets: - model-api.model-api.svc.cluster.local:8000 # 다른 네임스페이스에 있는 model-api의 클러스터 내부 주소와 포트 +alertmanager: + config: + global: + resolve_timeout: 5m # 알림이 해결된 것으로 간주하기 전에 기다리는 시간 5분 + route: + group_by: ["alertname"] # 같은 alertname을 가진 알림들을 그룹화 + group_wait: 30s # 첫 번째 알림이 발생한 후 그룹화된 알림을 전송하기 전에 기다리는 시간 30초 + group_interval: 5m # 그룹화된 알림을 전송한 후 다음 알림을 전송하기 전에 기다리는 시간 5분 + repeat_interval: 12h # 같은 알림이 계속 발생할 때 알림을 반복해서 전송하는 간격 12시간 + receiver: alert-bridge # alert-bridge로 알림 전송 + routes: + - matchers: + - alertname =~ "ChurnPrediction.*|ChurnConfidence.*|ChurnInference.*|ChurnFeature.*" + receiver: alert-bridge + receivers: + - name: alert-bridge + webhook_configs: + - url: http://alert-bridge.monitoring.svc.cluster.local:8080/alert # alert-bridge의 클러스터 내부 주소와 포트 + send_resolved: false # 알림이 해결되지 않은 경우에만 알림 전송 + grafana: ingress: enabled: true diff --git a/services/alert-bridge/Dockerfile b/services/alert-bridge/Dockerfile new file mode 100644 index 0000000..6efd114 --- /dev/null +++ b/services/alert-bridge/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/services/alert-bridge/main.py b/services/alert-bridge/main.py new file mode 100644 index 0000000..570c987 --- /dev/null +++ b/services/alert-bridge/main.py @@ -0,0 +1,36 @@ +import os +import httpx +from fastapi import FastAPI, Request + +app = FastAPI(title="Alert Bridge") + +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") +GITHUB_REPO_OWNER = os.getenv("GITHUB_REPO_OWNER") +GITHUB_REPO_NAME = os.getenv("GITHUB_REPO_NAME") + +# Alertmanager에서 알림을 받아서 GitHub Actions로 전달하는 역할을 하는 api 엔드포인트(Alertmanager가 요청을 보냄) +@app.post("/alert") +async def receive_alert(request: Request): + payload = await request.json() + + # alertmanager에서 받은 알림 중에서 firing 상태인 알림만 필터링 + firing_alerts = [a for a in payload.get("alerts", []) if a["status"] == "firing"] + if not firing_alerts: + return {"message": "no firing alerts"} + + alert_names = [a["labels"].get("alertname", "") for a in firing_alerts] + + async with httpx.AsyncClient() as client: + response = await client.post( + f"https://api.github.com/repos/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/dispatches", + headers={ + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github+json", + }, + json={ + "event_type": "model-retrain", + "client_payload": {"alerts": alert_names}, + }, + ) + + return {"status": response.status_code, "alerts": alert_names} diff --git a/services/alert-bridge/requirements.txt b/services/alert-bridge/requirements.txt new file mode 100644 index 0000000..d23d558 --- /dev/null +++ b/services/alert-bridge/requirements.txt @@ -0,0 +1,3 @@ +fastapi +uvicorn +httpx diff --git a/services/model-api/app/services/churn_service.py b/services/model-api/app/services/churn_service.py index b3e7ef4..5315553 100644 --- a/services/model-api/app/services/churn_service.py +++ b/services/model-api/app/services/churn_service.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from app.models.loader import load_churn_model -from prometheus_client import Counter, Histogram +from prometheus_client import Counter, Gauge, Histogram MODEL_NAME = "churn" @@ -18,16 +18,31 @@ 'cart_abandonment_rate', ] -prediction_counter = Counter( +# Prometheus 메트릭 정의 +# 총 예측 횟수, 예측된 클래스별로 레이블(predicted) 추가 +prediction_counter = Counter( # Counter는 총 횟수를 세는 메트릭 "churn_predictions_total", "이탈 예측 횟수", ["predicted"], ) -confidence_histogram = Histogram( +# 예측 신뢰도 분포 +confidence_histogram = Histogram( # Histogram은 값의 분포를 나타내는 메트릭 "churn_confidence_score", "이탈 예측 신뢰도 분포", buckets=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], ) +# 입력 피처의 평균값 +feature_mean_gauge = Gauge( # Gauge는 현재 값을 나타내는 메트릭 + "churn_feature_mean", + "입력 피처 평균값", + ["feature"], +) +# 추론 지연 시간 +inference_histogram = Histogram( + "churn_inference_duration_seconds", + "추론 지연 시간", + buckets=[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0], +) def predict(data: dict) -> dict: @@ -39,14 +54,19 @@ def predict(data: dict) -> dict: x = [[data[f] for f in FEATURES]] + for feature in FEATURES: + feature_mean_gauge.labels(feature=feature).set(data[feature]) + start = time.time() predicted = int(model.predict(x)[0]) proba = model.predict_proba(x)[0] confidence = float(proba[predicted]) - time_ms = round((time.time() - start) * 1000, 3) + duration = time.time() - start + time_ms = round(duration * 1000, 3) prediction_counter.labels(predicted=str(predicted)).inc() confidence_histogram.observe(confidence) + inference_histogram.observe(duration) return { "result": {