Orpheus-FastAPI/docker-compose.yml at main · timonharz/Orpheus-FastAPI · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
version: "3.9"

services:
  orpheus-fastapi:
    container_name: orpheus-fastapi
    build:
      context: .
      dockerfile: Dockerfile.gpu
    ports:
      - "5005:5005"
    env_file:
      - .env
    environment:
      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
      - CUDA_VISIBLE_DEVICES=0
      - PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:2048,expandable_segments:True
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    shm_size: '8gb'
    ulimits:
      memlock:
        soft: -1
        hard: -1
    restart: unless-stopped
    depends_on:
      llama-cpp-server:
        condition: service_started

  llama-cpp-server:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda
    container_name: llama-cpp-server
    ports:
      - "5006:5006"
    volumes:
      - ./models:/models
    env_file:
      - .env
    environment:
      # Force use of cuBLAS for faster matmuls
      - GGML_CUDA_FORCE_CUBLAS=1
      - CUDA_VISIBLE_DEVICES=0
      - GGML_CUDA_FORCE_MMQ=1
      - GGML_CUDA_DMMV_X=1024
      - GGML_CUDA_MMV_Y=1024
      - GGML_CUDA_FORCE_F16=1
            # CUDA tuning
      - CUDA_VISIBLE_DEVICES=0
      - GGML_CUDA_PEER_MAX_BATCH_SIZE=8000
      - LLAMA_CUDA_HOST_BUFFER=16384         # increase host–GPU buffer (MiB)
      # Memory management
      - LLAMA_MMAP=1
      - GGML_CUDA_SPLIT_TENSORS=1  # Add this for more efficient tensor splitting
      - GGML_TENSOR_SPLIT_MODE=row  # Add this for better tensor distribution
      - GGML_CUDA_COMPUTE_CAPABILITY=8
      - GGML_CUDA_NO_FUSED_MLP=0


    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    shm_size: '24gb'
    ulimits:
      memlock:
        soft: -1
        hard: -1
    restart: unless-stopped
    command: >
      -m /models/${ORPHEUS_MODEL_NAME}
      --host ${LLAMA_ARG_HOST:-0.0.0.0}
      --port ${LLAMA_ARG_PORT:-5006}
      --n-gpu-layers 29
      --threads 6
      --batch-size 8192
      --ubatch-size 8192
      --ctx-size 16834
      --cont-batching
      --timeout 100
      --mlock
      --flash-attn
      --parallel 8
      --numa numactl
      --threads-http 2


  model-init:
    image: curlimages/curl:latest
    user: ${UID}:${GID}
    volumes:
      - ./models:/app/models
    working_dir: /app
    command: >
      sh -c '
      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
        echo "Downloading model file..."
        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
      else
        echo "Model file already exists"
      fi'
    restart: "no"