🚀 Deploy: 고객 이탈 예측 모델 학습 및 커스텀 도메인 기반 서비스 인프라 구축

.github/workflows/cd-production.yaml

@@ -24,7 +24,7 @@ jobs:
         run: kubectl apply -k k8s/model-api/overlays/production/
 
       - name: 파드 재시작 # (이미지 태그가 계속 latest여도 새 이미지로 배포되도록)
-        run: kubectl rollout restart deployment/model-api
+        run: kubectl rollout restart deployment/model-api -n model-api # 네임스페이스 지정
 
       - name: 배포 확인
-        run: kubectl rollout status deployment/model-api
+        run: kubectl rollout status deployment/model-api -n model-api # 네임스페이스 지정

k8s/cert-manager/cluster-issuer.yaml

@@ -0,0 +1,14 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer # 인증서 발급, 클러스터 전체에서 사용 가능
+metadata:
+  name: swmlops-issuer
+spec:
+  acme: # acme 프로토콜이 Let’s Encrypt와 통신하기 위한 표준 프로토콜
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: simgpt02@gmail.com
+    privateKeySecretRef:
+      name: swmlops-issuer
+    solvers:
+      - http01:
+          ingress:
+            ingressClassName: traefik # k3s 에서 기본 제공해주는 ingress controller

k8s/mlflow-db/deployment.yaml

@@ -2,6 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: mlflow-db
+  namespace: mlflow
 spec:
   replicas: 1
   selector:

k8s/mlflow-db/pvc.yaml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: mlflow-db-pvc
+  namespace: mlflow
 spec:
   accessModes:
     - ReadWriteOnce

k8s/mlflow-db/service.yaml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: mlflow-db
+  namespace: mlflow
 spec:
   type: ClusterIP
   selector:

k8s/mlflow/deployment.yaml

@@ -2,6 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: mlflow
+  namespace: mlflow
 spec:
   replicas: 1
   selector:
@@ -14,6 +15,7 @@ spec:
     spec:
       containers:
         - name: mlflow
+          image: burakince/mlflow:3.7.0 # postgreSQL 연결이 되는 버전 이미지
           image: burakince/mlflow:3.7.0 # postgreSQL 연결이 되는 버전 이미지
           ports:
             - containerPort: 5000
@@ -39,5 +41,5 @@ spec:
             - --port=5000
             - --default-artifact-root=s3://kube-ml-bucket/mlflow-artifacts # 모델 저장
             - --backend-store-uri=postgresql://mlflow:1234@mlflow-db:5432/mlflow
-            - --allowed-hosts=mlflow,mlflow:5000,mlflow.default.svc.cluster.local,localhost,localhost:5000 # 실제 운영 환경에서는 실제 도메인을 입력하면 될 듯
+            - --allowed-hosts=mlflow.mlflow.svc.cluster.local,localhost,localhost:5000,mlflow.swmlops.site # 풀 DNS, localhost, 도메인 허용
 

k8s/mlflow/ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: mlflow
+  namespace: mlflow
+  annotations:
+    cert-manager.io/cluster-issuer: swmlops-issuer # cert-manager에서 인증서를 발급받을 때 사용할 ClusterIssuer 지정
+spec:
+  ingressClassName: traefik # k3s 에서 기본 제공해주는 ingress controller
+  tls:
+    - hosts:
+        - mlflow.swmlops.site # 이 도메인으로 접속할 때 TLS 인증서 사용
+      secretName: mlflow-tls
+  rules:
+    - host: mlflow.swmlops.site # 이 도메인으로 접속할 때 아래의 http 경로로 라우팅
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: mlflow # mlflow 서비스로 이동 + 5000 포트로 이동
+                port:
+                  number: 5000

k8s/mlflow/service.yaml

@@ -2,11 +2,11 @@ apiVersion: v1
 kind: Service
 metadata:
   name: mlflow
+  namespace: mlflow
 spec:
-  type: NodePort
+  type: ClusterIP
   selector:
     app: mlflow
   ports:
     - port: 5000
       targetPort: 5000
-      nodePort: 30050

k8s/model-api/base/deployment.yaml

@@ -14,12 +14,13 @@ spec:
     spec:
       containers:
         - name: model-api
+          image: simgpt/model-api:latest # 이렇게 되어있어도 어차피 cd시에 태그가 커밋 sha로 정확한 버전 추적하도록 해놓음
           image: simgpt/model-api:latest # 이렇게 되어있어도 어차피 cd시에 태그가 커밋 sha로 정확한 버전 추적하도록 해놓음
           ports:
             - containerPort: 8000
           env:
             - name: MLFLOW_TRACKING_URI
-              value: http://mlflow:5000
+              value: http://mlflow.mlflow.svc.cluster.local:5000
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:
@@ -32,6 +33,8 @@ spec:
                   key: secret-access-key
             - name: AWS_DEFAULT_REGION
               value: ap-northeast-2
+            - name: CHURN_MODEL_VERSION # 모델 버전 추가
+              value: "v5"
           resources:
             requests:
               cpu: "250m"

k8s/model-api/overlays/local/kustomization.yaml

@@ -1,6 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: model-api
+
 resources:
   - ../../base     # base 가져다 쓴다
   - service.yaml

k8s/model-api/overlays/production/ingress.yaml

@@ -2,9 +2,18 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: model-api
+  namespace: model-api
+  annotations:
+    cert-manager.io/cluster-issuer: swmlops-issuer # cert-manager에서 인증서를 발급받을 때 사용할 ClusterIssuer 지정
 spec:
+  ingressClassName: traefik # k3s 에서 기본 제공해주는 ingress controller
+  tls:
+    - hosts:
+        - model.swmlops.site # 이 도메인으로 접속할 때 TLS 인증서 사용
+      secretName: model-api-tls
   rules:
-    - http:
+    - host: model.swmlops.site # 이 도메인으로 접속할 때 아래의 http 경로로 라우팅
+      http:
         paths:
           - path: /
             pathType: Prefix

k8s/model-api/overlays/production/kustomization.yaml

@@ -1,6 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: model-api # 네임스페이스 추가(일부러 deployment.yaml에서 네임스페이스 안쓰고 여기서만 씀)
+
 resources:
   - ../../base
   - service.yaml

k8s/monitoring/values.yaml

@@ -4,4 +4,17 @@ prometheus:
       - job_name: model-api
         static_configs:
           - targets:
-              - model-api.default.svc.cluster.local:8000
+              - model-api.model-api.svc.cluster.local:8000 # 다른 네임스페이스에 있는 model-api의 클러스터 내부 주소와 포트
+
+grafana:
+  ingress:
+    enabled: true
+    ingressClassName: traefik
+    annotations:
+      cert-manager.io/cluster-issuer: swmlops-issuer
+    hosts:
+      - monitoring.swmlops.site
+    tls:
+      - secretName: grafana-tls
+        hosts:
+          - monitoring.swmlops.site

k8s/namespaces.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mlflow
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: model-api
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring

services/model-api/app/main.py

@@ -17,10 +17,10 @@ def health():
 
 # MNIST 데이터 모델 예측 API
 @app.post("/mnist_predict", response_model=ResponseSchema)
-def predict(request: RequestSchema):
+def mnist_predict(request: RequestSchema):
     try:
-        service = get_service(request.type) # 모델에 맞는 service 지정
-        output = service.predict(request.data)  # 예측 결과
+        service = get_service(request.type)
+        output = service.predict(request.data)
 
         return ResponseSchema(
             success=True,
@@ -37,3 +37,27 @@ def predict(request: RequestSchema):
             metadata=None,
             error=str(e),
         )
+
+
+# 고객 이탈 예측 API
+@app.post("/churn_predict", response_model=ResponseSchema)
+def churn_predict(request: RequestSchema):
+    try:
+        service = get_service("churn")
+        output = service.predict(request.data)
+
+        return ResponseSchema(
+            success=True,
+            model="churn",
+            result=output["result"],
+            metadata=output["metadata"],
+            error=None,
+        )
+    except Exception as e:
+        return ResponseSchema(
+            success=False,
+            model="churn",
+            result=None,
+            metadata=None,
+            error=str(e),
+        )

services/model-api/app/models/loader.py

@@ -1,16 +1,17 @@
 import os
 import mlflow.pytorch
+import mlflow.sklearn
 
 
 _model_cache: dict = {}
 
-
+# mlflow에서 mnist 모델 로드하는 함수
 def load_model(model_name: str, version: str):
     cache_key = f"{model_name}_{version}"
     if cache_key in _model_cache:
         return _model_cache[cache_key]
 
-    mlflow_uri = os.getenv("MLFLOW_TRACKING_URI", "http://mlflow:5000")
+    mlflow_uri = os.getenv("MLFLOW_TRACKING_URI", "http://mlflow:5000") # 환경변수에서 mlflow tracking uri 가져오기, 없으면 기본값으로 http://mlflow:5000 사용
     mlflow.set_tracking_uri(mlflow_uri)
 
     model_uri = f"models:/{model_name}-{version}/latest"
@@ -19,3 +20,18 @@ def load_model(model_name: str, version: str):
 
     _model_cache[cache_key] = model
     return model
+
+# mlflow에서 고객 이탈 예측 모델 로드하는 함수
+def load_churn_model(model_name: str, version: str):
+    cache_key = f"{model_name}_{version}_sklearn"
+    if cache_key in _model_cache:
+        return _model_cache[cache_key]
+
+    mlflow_uri = os.getenv("MLFLOW_TRACKING_URI", "http://mlflow:5000")
+    mlflow.set_tracking_uri(mlflow_uri)
+
+    model_uri = f"models:/{model_name}-{version}/latest"
+    model = mlflow.sklearn.load_model(model_uri)
+
+    _model_cache[cache_key] = model
+    return model

services/model-api/app/services/churn_service.py

@@ -0,0 +1,65 @@
+import os
+import time
+from datetime import datetime, timezone
+
+from app.models.loader import load_churn_model
+from prometheus_client import Counter, Histogram
+
+MODEL_NAME = "churn"
+MODEL_VERSION = os.getenv("CHURN_MODEL_VERSION", "v5")
+
+FEATURES = [
+    'account_age_months',
+    'avg_order_value',
+    'total_orders',
+    'days_since_last_purchase',
+    'discount_usage_rate',
+    'return_rate',
+    'browsing_frequency_per_week',
+    'cart_abandonment_rate',
+]
+
+prediction_counter = Counter(
+    "churn_predictions_total",
+    "이탈 예측 횟수",
+    ["predicted"],
+)
+confidence_histogram = Histogram(
+    "churn_confidence_score",
+    "이탈 예측 신뢰도 분포",
+    buckets=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
+)
+
+
+def predict(data: dict) -> dict:
+    missing = [f for f in FEATURES if f not in data]
+    if missing:
+        raise ValueError(f"누락된 피처: {missing}")
+
+    model = load_churn_model(MODEL_NAME, MODEL_VERSION)
+
+    x = [[data[f] for f in FEATURES]]
+
+    start = time.time()
+    predicted = int(model.predict(x)[0])
+    proba = model.predict_proba(x)[0]
+    confidence = float(proba[predicted])
+    time_ms = round((time.time() - start) * 1000, 3)
+
+    prediction_counter.labels(predicted=str(predicted)).inc()
+    confidence_histogram.observe(confidence)
+
+    return {
+        "result": {
+            "predicted_class": predicted,
+            "churned": bool(predicted),
+            "confidence": confidence,
+            "probabilities": {"retained": float(proba[0]), "churned": float(proba[1])},
+        },
+        "metadata": {
+            "model": MODEL_NAME,
+            "version": MODEL_VERSION,
+            "inference_time_ms": time_ms,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+        },
+    }

services/model-api/app/services/service_factory.py

@@ -1,8 +1,9 @@
-from app.services import mnist_service
+from app.services import mnist_service, churn_service
 
 
 _services = {
     "mnist": mnist_service,
+    "churn": churn_service,
 }
 
 # 모델의 종류에 맞게 service 파일 불러오는 라우터