From 41cf995f0151c9c1cc80f51ec7ef17e7cf48bce1 Mon Sep 17 00:00:00 2001 From: Pat Date: Thu, 12 Feb 2026 11:42:50 -0600 Subject: [PATCH 1/4] Reorganize the truss-examples folder --- .claude/CLAUDE.md | 127 ++++ .droid.yaml | 62 -- .github/actions/setup-python/action.yml | 34 - .github/workflows/pr.yml | 8 +- .github/workflows/test-examples.yml | 19 +- .github/workflows/truss_deploy.yml | 2 +- .github/workflows/warm-chains.yml | 6 +- .gitignore | 157 +---- .isort.cfg | 2 - .pre-commit-config.yaml | 11 +- .tool-versions | 2 - .../README.md | 29 - .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 185 ----- .../README.md | 185 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 187 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../BEI-Bert-tanaos-tanaos-ner-v1/README.md | 187 ----- .../BEI-Bert-taylorai-bge-micro-v2/README.md | 178 ----- .../BEI-Bert-voyageai-voyage-4-nano/README.md | 178 ----- .../README.md | 169 ----- .../README.md | 181 ----- .../README.md | 178 ----- .../BEI-baai-bge-m3-embedding-dense/README.md | 178 ----- .../README.md | 178 ----- .../BEI-baai-bge-reranker-large/README.md | 185 ----- .../README.md | 185 ----- .../README.md | 169 ----- .../README.md | 181 ----- .../README.md | 181 ----- .../README.md | 181 ----- .../README.md | 178 ----- .../README.md | 169 ----- .../README.md | 169 ----- .../README.md | 185 ----- .../README.md | 181 ----- .../README.md | 166 ----- .../README.md | 181 ----- .../BEI-qwen-qwen3-embedding-4b-fp4/README.md | 180 ----- .../BEI-qwen-qwen3-embedding-4b-fp8/README.md | 181 ----- .../BEI-qwen-qwen3-embedding-8b-fp8/README.md | 181 ----- .../README.md | 169 ----- .../BEI-qwen-qwen3-reranker-4b-fp8/README.md | 169 ----- .../BEI-qwen-qwen3-reranker-8b-fp4/README.md | 168 ----- .../BEI-qwen-qwen3-reranker-8b-fp8/README.md | 169 ----- .../README.md | 181 ----- .../README.md | 166 ----- .../README.md | 169 ----- .../README.md | 178 ----- .../README.md | 178 ----- .../README.md | 168 ----- .../README.md | 168 ----- .../README.md | 169 ----- .../README.md | 168 ----- .../README.md | 168 ----- .../BISV2-nvidia-qwen3-30b-a3b-fp4/README.md | 168 ----- .../BISV2-nvidia-qwen3-8b-fp4/README.md | 168 ----- .../README.md | 171 ----- .../README.md | 171 ----- .../BISV2-qwen-qwen3-32b-fp4/README.md | 168 ----- .../BISV2-qwen-qwen3-4b-fp8/README.md | 169 ----- .../README.md | 171 ----- .../README.md | 173 ----- .../Briton-google-gemma-3-1b-it/README.md | 169 ----- .../Briton-google-gemma-3-270m-it/README.md | 169 ----- .../README.md | 178 ----- .../Briton-google-gemma-3-27b-it/README.md | 169 ----- .../README.md | 171 ----- .../README.md | 181 ----- .../README.md | 173 ----- .../README.md | 174 ----- .../README.md | 172 ----- .../README.md | 168 ----- .../README.md | 169 ----- .../README.md | 171 ----- .../README.md | 171 ----- .../Briton-microsoft-phi-4-fp8/README.md | 171 ----- .../README.md | 168 ----- .../README.md | 172 ----- .../README.md | 172 ----- .../README.md | 182 ----- .../README.md | 172 ----- .../README.md | 182 ----- .../README.md | 181 ----- .../Briton-qwen-qwen3-30b-a3b-fp8/README.md | 172 ----- .../README.md | 172 ----- .../Briton-qwen-qwen3-30b-a3b/README.md | 168 ----- .../README.md | 170 ----- .../Briton-qwen-qwen3-32b-fp4/README.md | 170 ----- .../Briton-qwen-qwen3-32b-fp8/README.md | 173 ----- .../README.md | 181 ----- .../README.md | 173 ----- .../README.md | 182 ----- .../README.md | 171 ----- .../README.md | 202 ------ .../README.md | 202 ------ .../README.md | 202 ------ .../README.md | 209 ------ .../TEI-baai-bge-reranker-large/README.md | 209 ------ .../TEI-google-embeddinggemma-300m/README.md | 202 ------ .../README.md | 202 ------ .../README.md | 202 ------ .../README.md | 202 ------ .../README.md | 202 ------ .../README.md | 202 ------ .../README.md | 202 ------ .../TEI-redis-langcache-embed-v2/README.md | 202 ------ .../README.md | 202 ------ .../TEI-taylorai-bge-micro-v2/README.md | 202 ------ CONTRIBUTING.md | 101 +-- README.md | 188 +++++- _archive/README.md | 11 + .../deepspeed-mii}/README.md | 0 .../deepspeed-mii}/config.yaml | 0 .../deepspeed-mii}/model/__init__.py | 0 .../deepspeed-mii}/model/model.py | 0 .../model_cach_gcs}/config.yaml | 0 .../model_cach_gcs}/model/model.py | 0 .../nous-capybara-34b-openai/README.md | 0 .../nous-capybara-34b-openai/config.yaml | 0 .../model/__init__.py | 0 .../nous-capybara-34b-openai/model/model.py | 0 .../nous-capybara-34b/README.md | 0 .../nous-capybara-34b/config.yaml | 0 .../nous-capybara-34b}/model/__init__.py | 0 .../nous-capybara-34b/model/model.py | 0 {vllm => _archive/vllm}/README.md | 0 {vllm => _archive/vllm}/config.yaml | 0 .../vllm}/model/__init__.py | 0 .../vllm}/model/helper.py | 0 .../vllm}/model/model.py | 0 .../whisper-torchserve/README.md | 0 .../whisper-torchserve/config.yaml | 0 .../whisper-torchserve/data/config.properties | 0 .../whisper-torchserve}/model/__init__.py | 0 .../whisper-torchserve/model/model.py | 0 .../11-trt-original-README.md | 0 _internal/README.md | 116 ++++ .../assets}/comfyui-screenshot-1.png | Bin .../assets}/comfyui-screenshot-2.png | Bin .../assets}/comfyui-screenshot-3.png | Bin .../assets}/sdxl-controlnet-workflow.json | 0 .../deepseek-v3-0324/README.md | 0 .../deepseek-v3-0324/config.yaml | 0 .../glm47/config.yaml | 0 .../gpt-oss-120b/README.md | 0 .../gpt-oss-120b/config.yaml | 0 .../kimi-k2-instruct/README.md | 0 .../kimi-k2-instruct/config.yaml | 0 .../llama-4-maverick/config.yaml | 0 .../qwen3-Coder-30b-instruct/README.md | 0 .../qwen3-Coder-30b-instruct/config.yaml | 0 _internal/bin/generate_readmes.py | 639 ++++++++++++++++++ {bin => _internal/bin}/image.txt | 0 _internal/bin/test_all.py | 605 +++++++++++++++++ {bin => _internal/bin}/test_example.py | 0 {bin => _internal/bin}/test_truss_deploy.py | 2 +- _internal/bin/validate_all.py | 345 ++++++++++ {bin => _internal/bin}/validate_ci.py | 0 .../dockerfiles}/ComfyUI.dockerfile | 0 .../essential}/config.yaml | 0 {internal => _internal/internal}/config.yaml | 0 {templates => _internal/templates}/README.md | 0 .../faster-whisper-truss/config.yaml | 0 .../faster-whisper-truss}/model/__init__.py | 0 .../faster-whisper-truss/model/model.py | 0 .../templates}/generate.py | 0 .../templates}/generate.yaml | 0 .../config.yaml | 0 .../model/__init__.py | 0 .../model/model.py | 0 .../templates/trt-llm}/TRT-LLM-README.md | 0 .../templates}/trt-llm/config.yaml | 0 .../templates/trt-llm}/data/.gitattributes | 0 .../templates/trt-llm/model}/__init__.py | 0 .../templates/trt-llm}/model/model.py | 0 .../templates/trt-llm}/packages/client.py | 0 .../ensemble/config.pbtxt.jinja | 0 .../postprocessing/1/model.py | 0 .../postprocessing/config.pbtxt.jinja | 0 .../preprocessing/1/model.py | 0 .../preprocessing/config.pbtxt.jinja | 0 .../tensorrt_llm/config.pbtxt | 0 .../templates/trt-llm}/packages/utils.py | 0 .../templating/.internal_tei/Dockerfile | 0 .../.internal_tei/roll_out_docker.sh | 0 .../templating/README.md | 0 .../templating/deploy_all.py | 0 .../templating/generate_templates.py | 40 +- .../high_throughput/README.md | 0 .../high_throughput/config.yaml | 0 .../high_throughput}/model/__init__.py | 0 .../large_context/README.md | 0 .../large_context/config.yaml | 0 .../large_context}/model/__init__.py | 0 .../llama-3_1-70b-instruct/low_ttft/README.md | 0 .../low_ttft/config.yaml | 0 .../low_ttft}/model/__init__.py | 0 .../high_throughput/README.md | 0 .../high_throughput/config.yaml | 0 .../high_throughput}/model/__init__.py | 0 .../large_context/README.md | 0 .../large_context/config.yaml | 0 .../large_context/model}/__init__.py | 0 .../llama-3_1-8b-instruct/low_ttft/README.md | 0 .../low_ttft/config.yaml | 0 .../low_ttft/model}/__init__.py | 0 audio/README.md | 33 + audio/audiogen-medium/README.md | 36 + .../audiogen-medium}/config.yaml | 3 +- .../audiogen-medium/model}/__init__.py | 0 .../audiogen-medium}/model/model.py | 0 audio/chatterbox-tts/README.md | 31 + .../chatterbox-tts}/config.yaml | 6 +- .../chatterbox-tts}/docker/Dockerfile | 0 .../chatterbox-tts}/docker/docker_build.sh | 0 .../chatterbox-tts}/input/obama_8s.wav | Bin .../chatterbox-tts/model}/__init__.py | 0 .../chatterbox-tts}/model/model.py | 0 .../chatterbox-tts}/run_tts.py | 0 audio/kokoro/README.md | 34 + {kokoro => audio/kokoro}/call.py | 0 {kokoro => audio/kokoro}/config.yaml | 6 +- .../kokoro/model}/__init__.py | 0 {kokoro => audio/kokoro}/model/model.py | 0 audio/metavoice-1b/README.md | 32 + .../metavoice-1b}/config.yaml | 2 + .../metavoice-1b}/data/bria.mp3 | Bin .../metavoice-1b/model}/__init__.py | 0 .../metavoice-1b}/model/model.py | 0 .../metavoice-1b}/process.py | 0 .../metavoice-1b}/requirements.txt | 0 audio/musicgen-large/README.md | 36 + .../musicgen-large}/config.yaml | 3 +- .../musicgen-large/model}/__init__.py | 0 .../musicgen-large}/model/model.py | 0 audio/musicgen-melody/README.md | 36 + .../musicgen-melody}/config.yaml | 5 +- .../musicgen-melody/model}/__init__.py | 0 .../musicgen-melody}/model/model.py | 0 audio/nvidia-parakeet/README.md | 36 + .../nvidia-parakeet}/config.yaml | 4 +- .../nvidia-parakeet/model}/__init__.py | 0 .../nvidia-parakeet}/model/model.py | 0 audio/orpheus-3b-websockets/README.md | 40 ++ .../orpheus-3b-websockets}/call.py | 0 .../orpheus-3b-websockets}/config.yaml | 7 +- .../orpheus-3b-websockets}/model/model.py | 0 .../snac_batching_quantization_dev.py | 0 audio/orpheus-best-performance/README.md | 42 ++ .../orpheus-best-performance}/call.py | 0 .../orpheus-best-performance}/config.yaml | 1 + .../orpheus-best-performance}/model/model.py | 0 .../snac_batching_quantization_dev.py | 0 audio/piper-tts/README.md | 31 + {piper-tts => audio/piper-tts}/config.yaml | 2 + .../piper-tts/model}/__init__.py | 0 {piper-tts => audio/piper-tts}/model/model.py | 0 audio/qwen-asr/README.md | 49 ++ .../qwen-3-asr => audio/qwen-asr}/config.yaml | 12 +- audio/qwen-omni-thinker/README.md | 67 ++ .../qwen-omni-thinker}/config.yaml | 2 + audio/qwen-omni/README.md | 43 ++ .../qwen-omni}/config.yaml | 2 + .../qwen-omni}/model/model.py | 0 audio/sesame-csm-1b/README.md | 34 + .../sesame-csm-1b}/config.yaml | 8 +- .../sesame-csm-1b/model}/__init__.py | 0 .../sesame-csm-1b}/model/model.py | 0 audio/ultravox/README.md | 31 + {ultravox => audio/ultravox}/config.yaml | 5 +- .../ultravox/model}/__init__.py | 0 {ultravox => audio/ultravox}/model/model.py | 0 audio/voxtral-streaming-4b/README.md | 46 ++ .../voxtral-streaming-4b/config.yaml | 16 +- .../voxtral-streaming-4b/streaming.py | 0 audio/whisper/faster-whisper-small/README.md | 32 + .../whisper}/faster-whisper-small/config.yaml | 0 .../faster-whisper-small/model}/__init__.py | 0 .../faster-whisper-small/model/model.py | 0 audio/whisper/faster-whisper-v2/README.md | 32 + .../whisper}/faster-whisper-v2/config.yaml | 0 .../faster-whisper-v2}/model/__init__.py | 0 .../whisper}/faster-whisper-v2/model/model.py | 0 audio/whisper/faster-whisper-v3/README.md | 32 + .../whisper}/faster-whisper-v3/config.yaml | 0 .../faster-whisper-v3}/model/__init__.py | 0 .../whisper}/faster-whisper-v3/model/model.py | 0 .../whisper/whisper-openai}/LICENSE | 0 audio/whisper/whisper-openai/README.md | 31 + .../whisper/whisper-openai}/config.yaml | 1 + .../whisper/whisper-openai}/data/.gitkeep | 0 .../whisper/whisper-openai}/model/__init__.py | 0 .../whisper/whisper-openai}/model/model.py | 0 audio/whisper/whisper-streaming/README.md | 30 + .../whisper}/whisper-streaming/config.yaml | 3 + .../whisper-streaming}/model/__init__.py | 0 .../whisper}/whisper-streaming/model/model.py | 0 .../packages/whisper_streaming/line_packet.py | 0 .../whisper_streaming/whisper_online.py | 0 .../whisper_online_server.py | 0 .../whisper-streaming/requirements.txt | 0 audio/whisper/whisper-v3-base64/README.md | 31 + .../whisper/whisper-v3-base64}/config.yaml | 1 + .../whisper-v3-base64}/model/__init__.py | 0 .../whisper/whisper-v3-base64}/model/model.py | 0 audio/whisper/whisper-v3/README.md | 31 + .../whisper/whisper-v3}/config.yaml | 1 + .../whisper/whisper-v3}/model/__init__.py | 0 .../whisper/whisper-v3}/model/model.py | 0 audio/whisper/whisperx/README.md | 34 + .../whisper/whisperx}/config.yaml | 2 + .../whisper/whisperx}/model/__init__.py | 0 .../whisper/whisperx}/model/model.py | 0 audio/xtts-streaming/README.md | 29 + .../xtts-streaming}/config.yaml | 5 +- .../xtts-streaming}/model/__init__.py | 0 .../xtts-streaming}/model/model.py | 0 .../xtts-streaming}/requirements.txt | 0 audio/xtts-v2/README.md | 33 + {xtts-v2-truss => audio/xtts-v2}/config.yaml | 2 + .../xtts-v2}/model/__init__.py | 0 .../xtts-v2}/model/model.py | 0 audiogen-medium/README.md | 82 --- autodesk-wala/README.md | 75 -- chatterbox-tts/README.md | 13 - ci.yaml | 360 +++++++++- cogito/README.md | 117 ---- .../README.md | 104 --- .../README.md | 103 --- .../README.md | 104 --- .../README.md | 103 --- cogvlm/README.md | 48 -- comfyui-truss/README.md | 398 ----------- control-net-qrcode/README.md | 95 --- custom-engine-builder-control/README.md | 135 ---- custom-server/README.md | 9 - .../infinity-embedding-server/README.md | 80 --- custom-server/pixtral-12b/README.md | 85 --- custom-server/ultravox-0.4/README.md | 80 --- custom-server/ultravox-0.5-8b/README.md | 80 --- custom-server/ultravox-0.6-70b/README.md | 80 --- custom-server/voxtral-mini-3b-2507/README.md | 125 ---- .../voxtral-small-24b-2507/README.md | 125 ---- deepfloyd-xl/README.md | 83 --- deepseek-ocr/README.md | 81 --- .../README.md | 1 - .../README.md | 1 - .../README.md | 1 - .../README.md | 1 - .../README.md | 1 - dis-segmentation/README.md | 104 --- embeddings/README.md | 18 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 43 ++ .../config.yaml | 3 + .../baai-bge-en-icl-embedding-fp8/README.md | 31 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../bei/baai-bge-m3-embedding-dense/README.md | 30 + .../baai-bge-m3-embedding-dense}/config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../bei/baai-bge-reranker-large/README.md | 30 + .../bei/baai-bge-reranker-large}/config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 43 ++ .../config.yaml | 3 + .../README.md | 31 + .../config.yaml | 3 + .../bei/google-embeddinggemma-300m/README.md | 30 + .../google-embeddinggemma-300m}/config.yaml | 3 + .../README.md | 31 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 31 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 31 + .../config.yaml | 3 + .../README.md | 31 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../bei/ner-bert-base-ner-uncased/README.md | 30 + .../ner-bert-base-ner-uncased}/config.yaml | 3 + .../nomic-ai-nomic-embed-code-fp8/README.md | 31 + .../config.yaml | 3 + .../nomic-ai-nomic-embed-text-v1.5/README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../nvidia-llama-embed-nemotron-8b/README.md | 30 + .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 42 ++ .../config.yaml | 3 + .../qwen-qwen3-embedding-0.6b-fp8/README.md | 31 + .../config.yaml | 3 + .../bei/qwen-qwen3-embedding-4b-fp4/README.md | 31 + .../qwen-qwen3-embedding-4b-fp4}/config.yaml | 3 + .../bei/qwen-qwen3-embedding-4b-fp8/README.md | 31 + .../qwen-qwen3-embedding-4b-fp8}/config.yaml | 3 + .../bei/qwen-qwen3-embedding-8b-fp8/README.md | 31 + .../qwen-qwen3-embedding-8b-fp8}/config.yaml | 3 + .../qwen-qwen3-reranker-0.6b-fp8/README.md | 31 + .../qwen-qwen3-reranker-0.6b-fp8}/config.yaml | 3 + .../bei/qwen-qwen3-reranker-4b-fp8/README.md | 31 + .../qwen-qwen3-reranker-4b-fp8}/config.yaml | 3 + .../bei/qwen-qwen3-reranker-8b-fp4/README.md | 31 + .../qwen-qwen3-reranker-8b-fp4}/config.yaml | 3 + .../bei/qwen-qwen3-reranker-8b-fp8/README.md | 31 + .../qwen-qwen3-reranker-8b-fp8}/config.yaml | 3 + .../bei/redis-langcache-embed-v2/README.md | 30 + .../bei/redis-langcache-embed-v2}/config.yaml | 3 + .../README.md | 31 + .../config.yaml | 3 + .../README.md | 42 ++ .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + .../README.md | 43 ++ .../config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + embeddings/bei/tanaos-tanaos-ner-v1/README.md | 30 + .../bei/tanaos-tanaos-ner-v1}/config.yaml | 3 + .../bei/taylorai-bge-micro-v2/README.md | 30 + .../bei/taylorai-bge-micro-v2}/config.yaml | 3 + .../bei/voyageai-voyage-4-nano/README.md | 30 + .../bei/voyageai-voyage-4-nano}/config.yaml | 3 + .../README.md | 30 + .../config.yaml | 3 + embeddings/clip/README.md | 31 + {clip => embeddings/clip}/config.yaml | 6 +- .../clip}/model/__init__.py | 0 {clip => embeddings/clip}/model/model.py | 0 .../README.md | 32 + .../config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../tei/baai-bge-reranker-large/README.md | 32 + .../tei/baai-bge-reranker-large}/config.yaml | 1 + .../tei/google-embeddinggemma-300m/README.md | 32 + .../google-embeddinggemma-300m}/config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../nomic-ai-nomic-embed-text-v1.5/README.md | 32 + .../config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../tei/redis-langcache-embed-v2/README.md | 32 + .../tei/redis-langcache-embed-v2}/config.yaml | 1 + .../README.md | 32 + .../config.yaml | 1 + .../tei/taylorai-bge-micro-v2/README.md | 32 + .../tei/taylorai-bge-micro-v2}/config.yaml | 1 + .../text-embeddings-inference/README.md | 33 + .../text-embeddings-inference}/config.yaml | 3 +- falcon/falcon3-10B-trt-llm-spec-dec/README.md | 49 -- .../README.md | 49 -- flux.1-dev-trt-b200/README.md | 280 -------- flux/README.md | 82 --- flux/schnell/README.md | 81 --- fotographer/zenctrl/README.md | 17 - gemma/gemma-2-27b-it-vllm/README.md | 55 -- gemma/gemma-2-9b-it-vllm/README.md | 55 -- gfp-gan/README.md | 62 -- grpc/README.md | 74 -- image-segmentation/config.yaml | 13 - image/README.md | 29 + image/comfyui/README.md | 36 + {comfyui-truss => image/comfyui}/config.yaml | 3 +- .../comfyui}/data/comfy_ui_workflow.json | 0 .../comfyui}/data/model.json | 0 .../comfyui}/examples/animate-diff/model.json | 0 .../examples/animate-diff/workflow.json | 0 .../examples/anime-style-transfer/README.md | 31 + .../examples/anime-style-transfer/config.yaml | 10 +- .../anime-style-transfer/workflow.json | 0 .../examples/sdxl-controlnet/model.json | 0 .../examples/sdxl-controlnet/workflow.json | 0 .../examples/sdxl-with-refiner/model.json | 0 .../examples/sdxl-with-refiner/workflow.json | 0 .../comfyui/model}/__init__.py | 0 .../comfyui}/model/helpers.py | 0 .../comfyui}/model/model.py | 0 image/control-net-qrcode/README.md | 32 + .../control-net-qrcode}/config.yaml | 2 + .../controlnet_qr_code_results.gif | Bin .../control-net-qrcode/model}/__init__.py | 0 .../control-net-qrcode}/model/model.py | 0 .../control-net-qrcode}/twitter_mask.jpeg | Bin .../control-net-qrcode}/twitter_output.jpg | Bin image/deepfloyd-xl/README.md | 33 + .../deepfloyd-xl}/config.yaml | 23 +- .../deepfloyd-xl}/model/__init__.py | 0 .../deepfloyd-xl}/model/model.py | 0 image/dis-segmentation/README.md | 31 + .../dis-segmentation}/config.yaml | 2 + .../dis-segmentation}/model/__init__.py | 0 .../model/clone_repo_helper.py | 0 .../dis-segmentation}/model/helpers.py | 0 .../dis-segmentation}/model/model.py | 0 image/flux-dev-trt-b200/README.md | 34 + .../flux-dev-trt-b200}/config.yaml | 4 + .../flux-dev-trt-b200}/load_test.py | 0 .../flux-dev-trt-b200}/model/__init__.py | 0 .../model/demo_diffusion}/__init__.py | 0 .../model/demo_diffusion/dd_argparse.py | 0 .../model/demo_diffusion/dynamic_import.py | 0 .../model/demo_diffusion/engine.py | 0 .../model/demo_diffusion/image/__init__.py | 0 .../model/demo_diffusion/image/load.py | 0 .../model/demo_diffusion/image/resize.py | 0 .../model/demo_diffusion/image/video.py | 0 .../model/demo_diffusion/model/__init__.py | 0 .../model/demo_diffusion/model/base_model.py | 0 .../model/demo_diffusion/model/clip.py | 0 .../model/diffusion_transformer.py | 0 .../model/demo_diffusion/model/gan.py | 0 .../model/demo_diffusion/model/load.py | 0 .../model/demo_diffusion/model/lora.py | 0 .../model/demo_diffusion/model/optimizer.py | 0 .../model/demo_diffusion/model/scheduler.py | 0 .../model/demo_diffusion/model/t5.py | 0 .../model/demo_diffusion/model/tokenizer.py | 0 .../model/demo_diffusion/model/unet.py | 0 .../model/demo_diffusion/model/vae.py | 0 .../model/demo_diffusion/path/__init__.py | 0 .../model/demo_diffusion/path/dd_path.py | 0 .../model/demo_diffusion/path/resolve_path.py | 0 .../model/demo_diffusion/pipeline/__init__.py | 0 .../demo_diffusion/pipeline/calibrate.py | 0 .../pipeline/diffusion_pipeline.py | 0 .../demo_diffusion/pipeline/flux_pipeline.py | 0 .../pipeline/model_memory_manager.py | 0 .../pipeline/stable_cascade_pipeline.py | 0 .../pipeline/stable_diffusion_35_pipeline.py | 0 .../pipeline/stable_diffusion_3_pipeline.py | 0 .../pipeline/stable_diffusion_pipeline.py | 0 .../stable_video_diffusion_pipeline.py | 0 .../model/demo_diffusion/pipeline/type.py | 0 .../model/demo_diffusion/utils_modelopt.py | 0 .../demo_diffusion/utils_sd3}/__init__.py | 0 .../model/demo_diffusion/utils_sd3/mmdit.py | 0 .../demo_diffusion/utils_sd3/other_impls.py | 0 .../demo_diffusion/utils_sd3/sd3_impls.py | 0 .../flux-dev-trt-b200}/model/model.py | 0 .../flux-dev-trt-b200}/requirements.txt | 0 .../flux-dev-trt-b200}/show.py | 0 .../flux-dev-trt-b200}/show_batch.py | 0 image/flux/README.md | 46 ++ image/flux/dev/README.md | 34 + {flux => image/flux}/dev/config.yaml | 9 +- .../flux/dev}/model/__init__.py | 0 {flux => image/flux}/dev/model/model.py | 0 image/flux/schnell/README.md | 34 + {flux => image/flux}/schnell/config.yaml | 11 +- .../flux/schnell}/model/__init__.py | 0 {flux => image/flux}/schnell/model/model.py | 0 image/fotographer/zenctrl/README.md | 34 + .../fotographer}/zenctrl/call.py | 0 .../fotographer}/zenctrl/config.yaml | 8 +- .../fotographer}/zenctrl/images/banner_1.png | Bin .../fotographer}/zenctrl/images/camera.png | Bin .../zenctrl/images/speaker-input.png | Bin .../zenctrl/images/speaker-output.png | Bin .../fotographer}/zenctrl/requirements.txt | 0 .../whisper-truss => image/gfp-gan}/LICENSE | 0 image/gfp-gan/README.md | 31 + {gfp-gan => image/gfp-gan}/config.yaml | 3 + .../gfp-gan/data/.gitkeep | 0 {gfp-gan => image/gfp-gan}/input.json | 0 .../gfp-gan}/model/__init__.py | 0 {gfp-gan => image/gfp-gan}/model/model.py | 0 image/image-segmentation/README.md | 31 + image/image-segmentation/config.yaml | 19 + .../image-segmentation}/model/__init__.py | 0 .../image-segmentation}/model/model.py | 0 image/ip-adapter/README.md | 31 + {ip-adapter => image/ip-adapter}/config.yaml | 5 + .../ip-adapter}/model/__init__.py | 0 .../ip-adapter}/model/model.py | 0 image/magic-animate/README.md | 35 + .../magic-animate}/config.yaml | 2 + .../magic-animate}/model/__init__.py | 0 .../model/configs/inference/inference.yaml | 0 .../model/configs/prompts/animation.yaml | 0 .../magic-animate}/model/demo/animate.py | 0 .../magicanimate/models/appearance_encoder.py | 0 .../model/magicanimate/models/attention.py | 0 .../model/magicanimate/models/controlnet.py | 0 .../model/magicanimate/models/embeddings.py | 0 .../magicanimate/models/motion_module.py | 0 .../models/mutual_self_attention.py | 0 .../magicanimate/models/orig_attention.py | 0 .../model/magicanimate/models/resnet.py | 0 .../stable_diffusion_controlnet_reference.py | 0 .../model/magicanimate/models/unet.py | 0 .../magicanimate/models/unet_3d_blocks.py | 0 .../magicanimate/models/unet_controlnet.py | 0 .../model/magicanimate/pipelines/animation.py | 0 .../model/magicanimate/pipelines/context.py | 0 .../pipelines/pipeline_animation.py | 0 .../model/magicanimate/utils/dist_tools.py | 0 .../model/magicanimate/utils/util.py | 0 .../model/magicanimate/utils/videoreader.py | 0 .../magic-animate}/model/model.py | 0 image/playground-v2-aesthetic/README.md | 33 + .../playground-v2-aesthetic}/config.yaml | 0 .../model/__init__.py | 0 .../playground-v2-aesthetic}/model/model.py | 0 .../playground-v2-aesthetic}/show.py | 0 image/sana/sana-1600m/README.md | 40 ++ .../sana/sana-1600m}/config.yaml | 2 + .../sana/sana-1600m}/model/__init__.py | 0 .../sana/sana-1600m}/model/model.py | 0 .../sana-1600m}/packages/Sana/CITATION.bib | 0 .../packages/Sana/CIs/add_license_all.sh | 0 .../sana/sana-1600m}/packages/Sana/Dockerfile | 0 .../sana/sana-1600m}/packages/Sana/LICENSE | 0 .../sana/sana-1600m}/packages/Sana/README.md | 0 .../sana-1600m}/packages/Sana/app/app_sana.py | 0 .../packages/Sana/app/app_sana_multithread.py | 0 .../packages/Sana/app/safety_check.py | 0 .../packages/Sana/app/sana_pipeline.py | 0 .../sana-1600m}/packages/Sana/asset/Sana.jpg | Bin .../Sana/asset/docs/metrics_toolkit.md | 0 .../Sana/asset/example_data/00000000.png | Bin .../Sana/asset/example_data/00000000.txt | 0 .../example_data/00000000_InternVL2-26B.json | 0 .../00000000_InternVL2-26B_clip_score.json | 0 .../example_data/00000000_VILA1-5-13B.json | 0 .../00000000_VILA1-5-13B_clip_score.json | 0 .../00000000_prompt_clip_score.json | 0 .../Sana/asset/example_data/meta_data.json | 0 .../packages/Sana/asset/examples.py | 0 .../sana-1600m}/packages/Sana/asset/logo.png | Bin .../packages/Sana/asset/model-incremental.jpg | Bin .../packages/Sana/asset/model_paths.txt | 0 .../packages/Sana/asset/samples.txt | 0 .../packages/Sana/asset/samples_mini.txt | 0 .../sana_app_config/Sana_1600M_app.yaml | 0 .../sana_app_config/Sana_600M_app.yaml | 0 .../packages/Sana/configs/sana_base.yaml | 0 .../1024ms/Sana_1600M_img1024.yaml | 0 .../1024ms/Sana_1600M_img1024_AdamW.yaml | 0 .../sana_config/1024ms/Sana_600M_img1024.yaml | 0 .../sana_config/512ms/Sana_1600M_img512.yaml | 0 .../sana_config/512ms/Sana_600M_img512.yaml | 0 .../512ms/ci_Sana_600M_img512.yaml | 0 .../sana_config/512ms/sample_dataset.yaml | 0 .../packages/Sana/diffusion/__init__.py | 0 .../packages/Sana/diffusion/data/__init__.py | 0 .../packages/Sana/diffusion/data/builder.py | 0 .../Sana/diffusion/data/datasets/__init__.py | 0 .../Sana/diffusion/data/datasets/sana_data.py | 0 .../data/datasets/sana_data_multi_scale.py | 0 .../Sana/diffusion/data/datasets/utils.py | 0 .../Sana/diffusion/data/transforms.py | 0 .../Sana/diffusion/data/wids/__init__.py | 0 .../packages/Sana/diffusion/data/wids/wids.py | 0 .../Sana/diffusion/data/wids/wids_dl.py | 0 .../Sana/diffusion/data/wids/wids_lru.py | 0 .../Sana/diffusion/data/wids/wids_mmtar.py | 0 .../Sana/diffusion/data/wids/wids_specs.py | 0 .../Sana/diffusion/data/wids/wids_tar.py | 0 .../packages/Sana/diffusion/dpm_solver.py | 0 .../Sana/diffusion/flow_euler_sampler.py | 0 .../packages/Sana/diffusion/iddpm.py | 0 .../packages/Sana/diffusion/lcm_scheduler.py | 0 .../Sana/diffusion}/model/__init__.py | 0 .../packages/Sana/diffusion/model/act.py | 0 .../packages/Sana/diffusion/model/builder.py | 0 .../model/dc_ae/efficientvit}/__init__.py | 0 .../model/dc_ae/efficientvit/ae_model_zoo.py | 0 .../dc_ae/efficientvit/apps}/__init__.py | 0 .../model/dc_ae/efficientvit/apps/setup.py | 0 .../efficientvit/apps/trainer/__init__.py | 0 .../efficientvit/apps/trainer/run_config.py | 0 .../dc_ae/efficientvit/apps/utils/__init__.py | 0 .../dc_ae/efficientvit/apps/utils/dist.py | 0 .../dc_ae/efficientvit/apps/utils/ema.py | 0 .../dc_ae/efficientvit/apps/utils/export.py | 0 .../dc_ae/efficientvit/apps/utils/image.py | 0 .../dc_ae/efficientvit/apps/utils/init.py | 0 .../model/dc_ae/efficientvit/apps/utils/lr.py | 0 .../dc_ae/efficientvit/apps/utils/metric.py | 0 .../dc_ae/efficientvit/apps/utils/misc.py | 0 .../dc_ae/efficientvit/apps/utils/opt.py | 0 .../dc_ae/efficientvit/models}/__init__.py | 0 .../models/efficientvit/__init__.py | 0 .../efficientvit/models/efficientvit/dc_ae.py | 0 .../dc_ae/efficientvit/models/nn/__init__.py | 0 .../model/dc_ae/efficientvit/models/nn/act.py | 0 .../dc_ae/efficientvit/models/nn/drop.py | 0 .../dc_ae/efficientvit/models/nn/norm.py | 0 .../model/dc_ae/efficientvit/models/nn/ops.py | 0 .../efficientvit/models/nn/triton_rms_norm.py | 0 .../efficientvit/models/utils/__init__.py | 0 .../dc_ae/efficientvit/models/utils/list.py | 0 .../efficientvit/models/utils/network.py | 0 .../dc_ae/efficientvit/models/utils/random.py | 0 .../Sana/diffusion/model/diffusion_utils.py | 0 .../Sana/diffusion/model/dpm_solver.py | 0 .../Sana/diffusion/model/edm_sample.py | 0 .../diffusion/model/gaussian_diffusion.py | 0 .../Sana/diffusion/model/nets/__init__.py | 0 .../diffusion/model/nets/basic_modules.py | 0 .../nets/fastlinear/develop_triton_ffn.py | 0 .../nets/fastlinear/develop_triton_litemla.py | 0 .../model/nets/fastlinear/modules/__init__.py | 0 .../nets/fastlinear/modules/flash_attn.py | 0 .../model/nets/fastlinear/modules/lite_mla.py | 0 .../fastlinear/modules/mb_conv_pre_glu.py | 0 .../model/nets/fastlinear/modules/nn/act.py | 0 .../model/nets/fastlinear/modules/nn/conv.py | 0 .../model/nets/fastlinear/modules/nn/norm.py | 0 .../fastlinear/modules/triton_lite_mla.py | 0 .../fastlinear/modules/triton_lite_mla_fwd.py | 0 .../custom_autotune.py | 0 .../linear_relu_fwd.py | 0 .../modules/triton_lite_mla_kernels/mm.py | 0 .../triton_lite_mla_kernels/pad_vk_mm_fwd.py | 0 .../proj_divide_bwd.py | 0 .../triton_lite_mla_kernels/vk_mm_relu_bwd.py | 0 .../vk_q_mm_divide_fwd.py | 0 .../vk_q_mm_relu_bwd.py | 0 .../modules/triton_mb_conv_pre_glu.py | 0 .../depthwise_conv_fwd.py | 0 .../linear_glu_fwd.py | 0 .../modules/utils/compare_results.py | 0 .../modules/utils/custom_autotune.py | 0 .../nets/fastlinear/modules/utils/dtype.py | 0 .../fastlinear/modules/utils/export_onnx.py | 0 .../nets/fastlinear/modules/utils/model.py | 0 .../diffusion/model/nets/fastlinear/readme.md | 0 .../Sana/diffusion/model/nets/sana.py | 0 .../Sana/diffusion/model/nets/sana_U_shape.py | 0 .../model/nets/sana_U_shape_multi_scale.py | 0 .../Sana/diffusion/model/nets/sana_blocks.py | 0 .../diffusion/model/nets/sana_multi_scale.py | 0 .../model/nets/sana_multi_scale_adaln.py | 0 .../Sana/diffusion/model/nets/sana_others.py | 0 .../packages/Sana/diffusion/model/norms.py | 0 .../packages/Sana/diffusion/model/respace.py | 0 .../Sana/diffusion/model/sa_solver.py | 0 .../Sana/diffusion/model/timestep_sampler.py | 0 .../packages/Sana/diffusion/model/utils.py | 0 .../packages/Sana/diffusion/sa_sampler.py | 0 .../Sana/diffusion/sa_solver_diffusers.py | 0 .../Sana/diffusion/utils}/__init__.py | 0 .../Sana/diffusion/utils/checkpoint.py | 0 .../packages/Sana/diffusion/utils/config.py | 0 .../Sana/diffusion/utils/data_sampler.py | 0 .../Sana/diffusion/utils/dist_utils.py | 0 .../Sana/diffusion/utils/import_utils.py | 0 .../packages/Sana/diffusion/utils/logger.py | 0 .../Sana/diffusion/utils/lr_scheduler.py | 0 .../packages/Sana/diffusion/utils/misc.py | 0 .../Sana/diffusion/utils/optimizer.py | 0 .../packages/Sana/environment_setup.sh | 0 .../sana-1600m}/packages/Sana/pyproject.toml | 0 .../sana-1600m}/packages/Sana/sana/cli/run.py | 0 .../packages/Sana/sana/cli/upload2hf.py | 0 .../packages/Sana/sana/tools/__init__.py | 0 .../packages/Sana/sana/tools/download.py | 0 .../packages/Sana/sana/tools/hf_utils.py | 0 .../Sana/scripts/bash_run_inference_metric.sh | 0 .../scripts/bash_run_inference_metric_dpg.sh | 0 .../bash_run_inference_metric_geneval.sh | 0 .../bash_run_inference_metric_imagereward.sh | 0 .../infer_metric_run_inference_metric.sh | 0 ...fer_metric_run_inference_metric_geneval.sh | 0 .../Sana/scripts/infer_run_inference.sh | 0 .../scripts/infer_run_inference_geneval.sh | 0 .../infer_run_inference_geneval_diffusers.sh | 0 .../packages/Sana/scripts/inference.py | 0 .../packages/Sana/scripts/inference_dpg.py | 0 .../Sana/scripts/inference_geneval.py | 0 .../scripts/inference_geneval_diffusers.py | 0 .../Sana/scripts/inference_image_reward.py | 0 .../packages/Sana/scripts/interface.py | 0 .../packages/Sana/scripts/style.css | 0 .../packages/Sana/tests/bash/entry.sh | 0 .../Sana/tests/bash/test_inference.sh | 0 .../Sana/tests/bash/test_training_1epoch.sh | 0 .../packages/Sana/tools}/__init__.py | 0 .../packages/Sana/tools/convert_py_to_yaml.py | 0 .../tools/convert_sana_pag_to_diffusers.py | 0 .../Sana/tools/convert_sana_to_diffusers.py | 0 .../packages/Sana/tools/download.py | 0 .../Sana/tools/metrics/clip-score/.gitignore | 0 .../Sana/tools/metrics/clip-score/LICENSE | 0 .../Sana/tools/metrics/clip-score/README.md | 0 .../tools/metrics/clip-score/clip_score.py | 0 .../Sana/tools/metrics/clip-score/setup.py | 0 .../clip-score/src/clip_score/__init__.py | 0 .../clip-score/src/clip_score/__main__.py | 0 .../clip-score/src/clip_score/clip_score.py | 0 .../Sana/tools/metrics/compute_clipscore.sh | 0 .../Sana/tools/metrics/compute_dpg.sh | 0 .../tools/metrics/compute_fid_embedding.sh | 0 .../Sana/tools/metrics/compute_geneval.sh | 0 .../Sana/tools/metrics/compute_imagereward.sh | 0 .../metrics/dpg_bench/compute_dpg_bench.py | 0 .../tools/metrics/dpg_bench/dpg_bench.csv | 0 .../tools/metrics/dpg_bench/requirements.txt | 0 .../Sana/tools/metrics/geneval/LICENSE | 0 .../Sana/tools/metrics/geneval/README.md | 0 .../geneval/annotations/annotations_clip.csv | 0 .../geneval/annotations/annotations_if-xl.csv | 0 .../geneval/annotations/annotations_sdv2.csv | 0 .../annotations/mturk_hit_template.html | 0 .../tools/metrics/geneval/environment.yml | 0 .../geneval/evaluation/download_models.sh | 0 .../geneval/evaluation/evaluate_images.py | 0 .../geneval/evaluation/object_names.txt | 0 .../geneval/evaluation/summary_scores.py | 0 .../geneval/generation/diffusers_generate.py | 0 .../geneval/images/geneval_figure_1.png | 0 .../metrics/geneval/prompts/create_prompts.py | 0 .../geneval/prompts/evaluation_metadata.jsonl | 0 .../geneval/prompts/generation_prompts.txt | 0 .../metrics/geneval/prompts/object_names.txt | 0 .../image_reward/benchmark-prompts-dict.json | 0 .../image_reward/compute_image_reward.py | 0 .../Sana/tools/metrics/pytorch-fid/.gitignore | 0 .../tools/metrics/pytorch-fid/CHANGELOG.md | 0 .../Sana/tools/metrics/pytorch-fid/LICENSE | 0 .../Sana/tools/metrics/pytorch-fid/README.md | 0 .../tools/metrics/pytorch-fid/compute_fid.py | 0 .../Sana/tools/metrics/pytorch-fid/noxfile.py | 0 .../Sana/tools/metrics/pytorch-fid/setup.cfg | 0 .../Sana/tools/metrics/pytorch-fid/setup.py | 0 .../pytorch-fid/src/pytorch_fid/__init__.py | 0 .../pytorch-fid/src/pytorch_fid/__main__.py | 0 .../pytorch-fid/src/pytorch_fid/fid_score.py | 0 .../pytorch-fid/src/pytorch_fid/inception.py | 0 .../pytorch-fid/tests/test_fid_score.py | 0 .../packages/Sana/tools/metrics/utils.py | 0 .../packages/Sana/train_scripts/train.py | 0 .../packages/Sana/train_scripts/train.sh | 0 image/sana/sana-600m/README.md | 40 ++ .../sana/sana-600m}/config.yaml | 2 + .../sana/sana-600m}/model/__init__.py | 0 .../sana/sana-600m}/model/model.py | 0 .../sana-600m}/packages/Sana/CITATION.bib | 0 .../packages/Sana/CIs/add_license_all.sh | 0 .../sana/sana-600m}/packages/Sana/Dockerfile | 0 .../sana/sana-600m}/packages/Sana/LICENSE | 0 .../sana/sana-600m}/packages/Sana/README.md | 0 .../sana-600m}/packages/Sana/app/app_sana.py | 0 .../packages/Sana/app/app_sana_multithread.py | 0 .../packages/Sana/app/safety_check.py | 0 .../packages/Sana/app/sana_pipeline.py | 0 .../sana-600m}/packages/Sana/asset/Sana.jpg | Bin .../Sana/asset/docs/metrics_toolkit.md | 0 .../Sana/asset/example_data/00000000.png | Bin .../Sana/asset/example_data/00000000.txt | 0 .../example_data/00000000_InternVL2-26B.json | 0 .../00000000_InternVL2-26B_clip_score.json | 0 .../example_data/00000000_VILA1-5-13B.json | 0 .../00000000_VILA1-5-13B_clip_score.json | 0 .../00000000_prompt_clip_score.json | 0 .../Sana/asset/example_data/meta_data.json | 0 .../packages/Sana/asset/examples.py | 0 .../sana-600m}/packages/Sana/asset/logo.png | Bin .../packages/Sana/asset/model-incremental.jpg | Bin .../packages/Sana/asset/model_paths.txt | 0 .../packages/Sana/asset/samples.txt | 0 .../packages/Sana/asset/samples_mini.txt | 0 .../sana_app_config/Sana_1600M_app.yaml | 0 .../sana_app_config/Sana_600M_app.yaml | 0 .../packages/Sana/configs/sana_base.yaml | 0 .../1024ms/Sana_1600M_img1024.yaml | 0 .../1024ms/Sana_1600M_img1024_AdamW.yaml | 0 .../sana_config/1024ms/Sana_600M_img1024.yaml | 0 .../sana_config/512ms/Sana_1600M_img512.yaml | 0 .../sana_config/512ms/Sana_600M_img512.yaml | 0 .../512ms/ci_Sana_600M_img512.yaml | 0 .../sana_config/512ms/sample_dataset.yaml | 0 .../packages/Sana/diffusion/__init__.py | 0 .../packages/Sana/diffusion/data/__init__.py | 0 .../packages/Sana/diffusion/data/builder.py | 0 .../Sana/diffusion/data/datasets/__init__.py | 0 .../Sana/diffusion/data/datasets/sana_data.py | 0 .../data/datasets/sana_data_multi_scale.py | 0 .../Sana/diffusion/data/datasets/utils.py | 0 .../Sana/diffusion/data/transforms.py | 0 .../Sana/diffusion/data/wids/__init__.py | 0 .../packages/Sana/diffusion/data/wids/wids.py | 0 .../Sana/diffusion/data/wids/wids_dl.py | 0 .../Sana/diffusion/data/wids/wids_lru.py | 0 .../Sana/diffusion/data/wids/wids_mmtar.py | 0 .../Sana/diffusion/data/wids/wids_specs.py | 0 .../Sana/diffusion/data/wids/wids_tar.py | 0 .../packages/Sana/diffusion/dpm_solver.py | 0 .../Sana/diffusion/flow_euler_sampler.py | 0 .../packages/Sana/diffusion/iddpm.py | 0 .../packages/Sana/diffusion/lcm_scheduler.py | 0 .../Sana/diffusion}/model/__init__.py | 0 .../packages/Sana/diffusion/model/act.py | 0 .../packages/Sana/diffusion/model/builder.py | 0 .../model/dc_ae/efficientvit}/__init__.py | 0 .../model/dc_ae/efficientvit/ae_model_zoo.py | 0 .../dc_ae/efficientvit/apps}/__init__.py | 0 .../model/dc_ae/efficientvit/apps/setup.py | 0 .../efficientvit/apps/trainer/__init__.py | 0 .../efficientvit/apps/trainer/run_config.py | 0 .../dc_ae/efficientvit/apps/utils/__init__.py | 0 .../dc_ae/efficientvit/apps/utils/dist.py | 0 .../dc_ae/efficientvit/apps/utils/ema.py | 0 .../dc_ae/efficientvit/apps/utils/export.py | 0 .../dc_ae/efficientvit/apps/utils/image.py | 0 .../dc_ae/efficientvit/apps/utils/init.py | 0 .../model/dc_ae/efficientvit/apps/utils/lr.py | 0 .../dc_ae/efficientvit/apps/utils/metric.py | 0 .../dc_ae/efficientvit/apps/utils/misc.py | 0 .../dc_ae/efficientvit/apps/utils/opt.py | 0 .../dc_ae/efficientvit/models}/__init__.py | 0 .../models/efficientvit/__init__.py | 0 .../efficientvit/models/efficientvit/dc_ae.py | 0 .../dc_ae/efficientvit/models/nn/__init__.py | 0 .../model/dc_ae/efficientvit/models/nn/act.py | 0 .../dc_ae/efficientvit/models/nn/drop.py | 0 .../dc_ae/efficientvit/models/nn/norm.py | 0 .../model/dc_ae/efficientvit/models/nn/ops.py | 0 .../efficientvit/models/nn/triton_rms_norm.py | 0 .../efficientvit/models/utils/__init__.py | 0 .../dc_ae/efficientvit/models/utils/list.py | 0 .../efficientvit/models/utils/network.py | 0 .../dc_ae/efficientvit/models/utils/random.py | 0 .../Sana/diffusion/model/diffusion_utils.py | 0 .../Sana/diffusion/model/dpm_solver.py | 0 .../Sana/diffusion/model/edm_sample.py | 0 .../diffusion/model/gaussian_diffusion.py | 0 .../Sana/diffusion/model/nets/__init__.py | 0 .../diffusion/model/nets/basic_modules.py | 0 .../nets/fastlinear/develop_triton_ffn.py | 0 .../nets/fastlinear/develop_triton_litemla.py | 0 .../model/nets/fastlinear/modules/__init__.py | 0 .../nets/fastlinear/modules/flash_attn.py | 0 .../model/nets/fastlinear/modules/lite_mla.py | 0 .../fastlinear/modules/mb_conv_pre_glu.py | 0 .../model/nets/fastlinear/modules/nn/act.py | 0 .../model/nets/fastlinear/modules/nn/conv.py | 0 .../model/nets/fastlinear/modules/nn/norm.py | 0 .../fastlinear/modules/triton_lite_mla.py | 0 .../fastlinear/modules/triton_lite_mla_fwd.py | 0 .../custom_autotune.py | 0 .../linear_relu_fwd.py | 0 .../modules/triton_lite_mla_kernels/mm.py | 0 .../triton_lite_mla_kernels/pad_vk_mm_fwd.py | 0 .../proj_divide_bwd.py | 0 .../triton_lite_mla_kernels/vk_mm_relu_bwd.py | 0 .../vk_q_mm_divide_fwd.py | 0 .../vk_q_mm_relu_bwd.py | 0 .../modules/triton_mb_conv_pre_glu.py | 0 .../depthwise_conv_fwd.py | 0 .../linear_glu_fwd.py | 0 .../modules/utils/compare_results.py | 0 .../modules/utils/custom_autotune.py | 0 .../nets/fastlinear/modules/utils/dtype.py | 0 .../fastlinear/modules/utils/export_onnx.py | 0 .../nets/fastlinear/modules/utils/model.py | 0 .../diffusion/model/nets/fastlinear/readme.md | 0 .../Sana/diffusion/model/nets/sana.py | 0 .../Sana/diffusion/model/nets/sana_U_shape.py | 0 .../model/nets/sana_U_shape_multi_scale.py | 0 .../Sana/diffusion/model/nets/sana_blocks.py | 0 .../diffusion/model/nets/sana_multi_scale.py | 0 .../model/nets/sana_multi_scale_adaln.py | 0 .../Sana/diffusion/model/nets/sana_others.py | 0 .../packages/Sana/diffusion/model/norms.py | 0 .../packages/Sana/diffusion/model/respace.py | 0 .../Sana/diffusion/model/sa_solver.py | 0 .../Sana/diffusion/model/timestep_sampler.py | 0 .../packages/Sana/diffusion/model/utils.py | 0 .../packages/Sana/diffusion/sa_sampler.py | 0 .../Sana/diffusion/sa_solver_diffusers.py | 0 .../Sana/diffusion/utils}/__init__.py | 0 .../Sana/diffusion/utils/checkpoint.py | 0 .../packages/Sana/diffusion/utils/config.py | 0 .../Sana/diffusion/utils/data_sampler.py | 0 .../Sana/diffusion/utils/dist_utils.py | 0 .../Sana/diffusion/utils/import_utils.py | 0 .../packages/Sana/diffusion/utils/logger.py | 0 .../Sana/diffusion/utils/lr_scheduler.py | 0 .../packages/Sana/diffusion/utils/misc.py | 0 .../Sana/diffusion/utils/optimizer.py | 0 .../packages/Sana/environment_setup.sh | 0 .../sana-600m}/packages/Sana/pyproject.toml | 0 .../sana-600m}/packages/Sana/sana/cli/run.py | 0 .../packages/Sana/sana/cli/upload2hf.py | 0 .../packages/Sana/sana/tools/__init__.py | 0 .../packages/Sana/sana/tools/download.py | 0 .../packages/Sana/sana/tools/hf_utils.py | 0 .../Sana/scripts/bash_run_inference_metric.sh | 0 .../scripts/bash_run_inference_metric_dpg.sh | 0 .../bash_run_inference_metric_geneval.sh | 0 .../bash_run_inference_metric_imagereward.sh | 0 .../infer_metric_run_inference_metric.sh | 0 ...fer_metric_run_inference_metric_geneval.sh | 0 .../Sana/scripts/infer_run_inference.sh | 0 .../scripts/infer_run_inference_geneval.sh | 0 .../infer_run_inference_geneval_diffusers.sh | 0 .../packages/Sana/scripts/inference.py | 0 .../packages/Sana/scripts/inference_dpg.py | 0 .../Sana/scripts/inference_geneval.py | 0 .../scripts/inference_geneval_diffusers.py | 0 .../Sana/scripts/inference_image_reward.py | 0 .../packages/Sana/scripts/interface.py | 0 .../packages/Sana/scripts/style.css | 0 .../packages/Sana/tests/bash/entry.sh | 0 .../Sana/tests/bash/test_inference.sh | 0 .../Sana/tests/bash/test_training_1epoch.sh | 0 .../packages/Sana/tools}/__init__.py | 0 .../packages/Sana/tools/convert_py_to_yaml.py | 0 .../tools/convert_sana_pag_to_diffusers.py | 0 .../Sana/tools/convert_sana_to_diffusers.py | 0 .../packages/Sana/tools/download.py | 0 .../Sana/tools/metrics/clip-score/.gitignore | 0 .../Sana/tools/metrics/clip-score/LICENSE | 0 .../Sana/tools/metrics/clip-score/README.md | 0 .../tools/metrics/clip-score/clip_score.py | 0 .../Sana/tools/metrics/clip-score/setup.py | 0 .../clip-score/src/clip_score/__init__.py | 0 .../clip-score/src/clip_score/__main__.py | 0 .../clip-score/src/clip_score/clip_score.py | 0 .../Sana/tools/metrics/compute_clipscore.sh | 0 .../Sana/tools/metrics/compute_dpg.sh | 0 .../tools/metrics/compute_fid_embedding.sh | 0 .../Sana/tools/metrics/compute_geneval.sh | 0 .../Sana/tools/metrics/compute_imagereward.sh | 0 .../metrics/dpg_bench/compute_dpg_bench.py | 0 .../tools/metrics/dpg_bench/dpg_bench.csv | 0 .../tools/metrics/dpg_bench/requirements.txt | 0 .../Sana/tools/metrics/geneval/LICENSE | 0 .../Sana/tools/metrics/geneval/README.md | 0 .../geneval/annotations/annotations_clip.csv | 0 .../geneval/annotations/annotations_if-xl.csv | 0 .../geneval/annotations/annotations_sdv2.csv | 0 .../annotations/mturk_hit_template.html | 0 .../tools/metrics/geneval/environment.yml | 0 .../geneval/evaluation/download_models.sh | 0 .../geneval/evaluation/evaluate_images.py | 0 .../geneval/evaluation/object_names.txt | 0 .../geneval/evaluation/summary_scores.py | 0 .../geneval/generation/diffusers_generate.py | 0 .../geneval/images/geneval_figure_1.png | 0 .../metrics/geneval/prompts/create_prompts.py | 0 .../geneval/prompts/evaluation_metadata.jsonl | 0 .../geneval/prompts/generation_prompts.txt | 0 .../metrics/geneval/prompts/object_names.txt | 0 .../image_reward/benchmark-prompts-dict.json | 0 .../image_reward/compute_image_reward.py | 0 .../Sana/tools/metrics/pytorch-fid/.gitignore | 0 .../tools/metrics/pytorch-fid/CHANGELOG.md | 0 .../Sana/tools/metrics/pytorch-fid/LICENSE | 0 .../Sana/tools/metrics/pytorch-fid/README.md | 0 .../tools/metrics/pytorch-fid/compute_fid.py | 0 .../Sana/tools/metrics/pytorch-fid/noxfile.py | 0 .../Sana/tools/metrics/pytorch-fid/setup.cfg | 0 .../Sana/tools/metrics/pytorch-fid/setup.py | 0 .../pytorch-fid/src/pytorch_fid/__init__.py | 0 .../pytorch-fid/src/pytorch_fid/__main__.py | 0 .../pytorch-fid/src/pytorch_fid/fid_score.py | 0 .../pytorch-fid/src/pytorch_fid/inception.py | 0 .../pytorch-fid/tests/test_fid_score.py | 0 .../packages/Sana/tools/metrics/utils.py | 0 .../packages/Sana/train_scripts/train.py | 0 .../packages/Sana/train_scripts/train.sh | 0 image/segment-anything/README.md | 31 + .../segment-anything}/config.yaml | 2 + .../segment-anything}/model/__init__.py | 0 .../segment-anything}/model/model.py | 0 .../dreamshaper-lcm/README.md | 31 + .../dreamshaper-lcm/config.yaml | 21 + .../dreamshaper-lcm}/model/__init__.py | 0 .../dreamshaper-lcm/model/model.py | 0 .../playground-v2-trt/README.md | 35 + .../playground-v2-trt/config.yaml | 20 +- .../playground-v2-trt/model}/__init__.py | 0 .../playground-v2-trt/model/model.py | 0 .../packages/diffusion/trtclip.py | 0 .../packages/diffusion/trtunet.py | 0 .../playground-v2-trt/show.py | 0 .../sd-textual-inversion/README.md | 31 + .../sd-textual-inversion/config.yaml | 14 +- .../sd-textual-inversion/data/LulaCipher.bin | Bin .../sd-textual-inversion}/model/__init__.py | 0 .../sd-textual-inversion/model/model.py | 0 image/stable-diffusion/sd-turbo/README.md | 32 + .../stable-diffusion}/sd-turbo/config.yaml | 1 + .../sd-turbo}/model/__init__.py | 0 .../stable-diffusion}/sd-turbo/model/model.py | 0 .../sdxl-controlnet-canny/README.md | 31 + .../sdxl-controlnet-canny/baseten-logo.gif | Bin .../sdxl-controlnet-canny/config.yaml | 3 + .../sdxl-controlnet-canny}/model/__init__.py | 0 .../sdxl-controlnet-canny/model/model.py | 0 .../sdxl-controlnet-depth/README.md | 31 + .../sdxl-controlnet-depth/config.yaml | 3 + .../sdxl-controlnet-depth}/model/__init__.py | 0 .../sdxl-controlnet-depth/model/model.py | 0 .../sdxl-controlnet/README.md | 31 + .../sdxl-controlnet/baseten-logo.gif | Bin .../sdxl-controlnet/config.yaml | 13 +- .../sdxl-controlnet}/model/__init__.py | 0 .../sdxl-controlnet/model/model.py | 0 .../stable-diffusion/sdxl-lightning/README.md | 31 + .../sdxl-lightning/config.yaml | 2 + .../sdxl-lightning}/model/__init__.py | 0 .../sdxl-lightning/model/model.py | 0 .../sdxl-lora-swapping/README.md | 35 + .../sdxl-lora-swapping/config.yaml | 2 + .../sdxl-lora-swapping}/model/__init__.py | 0 .../sdxl-lora-swapping/model/model.py | 0 image/stable-diffusion/sdxl-lora/README.md | 31 + image/stable-diffusion/sdxl-lora/config.yaml | 25 + .../sdxl-lora}/model/__init__.py | 0 .../sdxl-lora/model/model.py | 0 image/stable-diffusion/sdxl-turbo/README.md | 32 + .../stable-diffusion}/sdxl-turbo/config.yaml | 1 + .../sdxl-turbo}/model/__init__.py | 0 .../sdxl-turbo/model/model.py | 0 .../stable-diffusion-3-medium/README.md | 35 + .../stable-diffusion-3-medium/config.yaml | 13 +- .../model/__init__.py | 0 .../stable-diffusion-3-medium/model/model.py | 0 .../stable-diffusion-inpainting-trt/README.md | 31 + .../config.yaml | 18 + .../model/__init__.py | 0 .../model/model.py | 0 .../packages/helpers/inpaint_pipeline.py | 0 .../packages/helpers/models.py | 0 .../helpers/stable_diffusion_pipeline.py | 0 .../packages/helpers/utilities.py | 0 .../requirements.txt | 0 .../README.md | 35 + .../config.yaml | 21 +- .../model/__init__.py | 0 .../model/model.py | 0 .../packages/diffusion/trtclip.py | 0 .../packages/diffusion/trtunet.py | 0 .../stable-diffusion-xl-1.0-trt-h100/show.py | 0 .../stable-diffusion-xl-1.0-trt/README.md | 35 + .../stable-diffusion-xl-1.0-trt/config.yaml | 21 +- .../model/__init__.py | 0 .../model/model.py | 0 .../packages/diffusion/trtclip.py | 0 .../packages/diffusion/trtunet.py | 0 .../stable-diffusion-xl-1.0-trt/show.py | 0 .../stable-diffusion-xl-1.0/README.md | 33 + .../stable-diffusion-xl-1.0/config.yaml | 0 .../model/__init__.py | 0 .../stable-diffusion-xl-1.0/model/model.py | 0 .../stable-diffusion-xl-1.0/show.py | 0 .../stable-diffusion/README.md | 31 + .../stable-diffusion/config.yaml | 17 +- .../stable-diffusion/data/model_index.json | 0 .../data/scheduler/scheduler_config.json | 0 .../data/text_encoder/config.json | 0 .../data/tokenizer/merges.txt | 0 .../data/tokenizer/special_tokens_map.json | 0 .../data/tokenizer/tokenizer_config.json | 0 .../data/tokenizer/vocab.json | 0 .../stable-diffusion/data/unet/config.json | 0 .../stable-diffusion/data/vae/config.json | 0 .../stable-diffusion}/model/__init__.py | 0 .../stable-diffusion/model/model.py | 0 .../stable-diffusion/show.py | 0 .../stable-video-diffusion/README.md | 31 + .../stable-video-diffusion/config.yaml | 3 + .../stable-video-diffusion}/model/__init__.py | 0 .../stable-video-diffusion/model/helper.py | 0 .../stable-video-diffusion/model/model.py | 0 .../model/scripts}/__init__.py | 0 .../model/scripts/sampling/configs/svd.yaml | 0 .../sampling/configs/svd_image_decoder.yaml | 0 .../scripts/sampling/configs/svd_xt.yaml | 0 .../configs/svd_xt_image_decoder.yaml | 0 .../scripts/sampling/simple_video_sample.py | 0 .../model/scripts/tests/attention.py | 0 .../model/scripts/util}/__init__.py | 0 .../model/scripts/util/detection}/__init__.py | 0 .../nsfw_and_watermark_dectection.py | 0 .../scripts/util/detection/p_head_v1.npz | Bin .../scripts/util/detection/w_head_v1.npz | Bin .../sample_images/cheetah.jpeg | Bin .../sample_images/racecar.jpeg | Bin infrastructure/README.md | 28 + infrastructure/autodesk-wala/README.md | 31 + .../autodesk-wala}/config.yaml | 4 + .../autodesk-wala}/examples/README.md | 0 .../examples/depth_maps_4/10.png | Bin .../examples/depth_maps_4/26.png | Bin .../examples/depth_maps_4/3.png | Bin .../examples/depth_maps_4/6.png | Bin .../examples/depth_maps_6/10.png | Bin .../examples/depth_maps_6/26.png | Bin .../examples/depth_maps_6/3.png | Bin .../examples/depth_maps_6/49.png | Bin .../examples/depth_maps_6/50.png | Bin .../examples/depth_maps_6/6.png | Bin .../examples/multi_view/003.png | Bin .../examples/multi_view/006.png | Bin .../examples/multi_view/010.png | Bin .../examples/multi_view/026.png | Bin .../examples/pointcloud/ring.h5df | Bin .../examples/single_depth_map/49.png | Bin .../examples/single_view/table.png | Bin .../autodesk-wala}/examples/sketch/tree.png | Bin .../examples/voxel/horse_16.json | 0 .../examples/voxel/horse_32.json | 0 .../autodesk-wala}/model/__init__.py | 0 .../autodesk-wala}/model/model.py | 0 .../autodesk-wala}/output.obj | 0 .../autodesk-wala/packages/src}/__init__.py | 0 .../packages/src/bpe_simple_vocab_16e6.txt.gz | Bin .../autodesk-wala}/packages/src/clip_mod.py | 0 .../packages/src/dataset_utils.py | 0 .../src/diffusion_modules}/__init__.py | 0 .../packages/src/diffusion_modules/dwt.py | 0 .../src/diffusion_modules/dwt_utils.py | 0 .../src/diffusion_modules/fp16_util.py | 0 .../diffusion_modules/gaussian_diffusion.py | 0 .../src/diffusion_modules/latent_points.py | 0 .../modules/__init__ copy.py | 0 .../src/diffusion_modules/modules/__init__.py | 0 .../diffusion_modules/modules/ball_query.py | 0 .../src/diffusion_modules/modules/frustum.py | 0 .../modules/functional/__init__.py | 0 .../modules/functional/backend.py | 0 .../modules/functional/ball_query.py | 0 .../modules/functional/devoxelization.py | 0 .../modules/functional/grouping.py | 0 .../modules/functional/interpolatation.py | 0 .../modules/functional/loss.py | 0 .../modules/functional/sampling.py | 0 .../functional/src/ball_query/ball_query.cpp | 0 .../functional/src/ball_query/ball_query.cu | 0 .../functional/src/ball_query/ball_query.cuh | 0 .../functional/src/ball_query/ball_query.hpp | 0 .../modules/functional/src/bindings.cpp | 0 .../modules/functional/src/cuda_utils.cuh | 0 .../functional/src/grouping/grouping.cpp | 0 .../functional/src/grouping/grouping.cu | 0 .../functional/src/grouping/grouping.cuh | 0 .../functional/src/grouping/grouping.hpp | 0 .../src/interpolate/neighbor_interpolate.cpp | 0 .../src/interpolate/neighbor_interpolate.cu | 0 .../src/interpolate/neighbor_interpolate.cuh | 0 .../src/interpolate/neighbor_interpolate.hpp | 0 .../src/interpolate/trilinear_devox.cpp | 0 .../src/interpolate/trilinear_devox.cu | 0 .../src/interpolate/trilinear_devox.cuh | 0 .../src/interpolate/trilinear_devox.hpp | 0 .../functional/src/sampling/sampling.cpp | 0 .../functional/src/sampling/sampling.cu | 0 .../functional/src/sampling/sampling.cuh | 0 .../functional/src/sampling/sampling.hpp | 0 .../modules/functional/src/utils.hpp | 0 .../functional/src/voxelization/vox.cpp | 0 .../functional/src/voxelization/vox.cu | 0 .../functional/src/voxelization/vox.cuh | 0 .../functional/src/voxelization/vox.hpp | 0 .../modules/functional/voxelization.py | 0 .../src/diffusion_modules/modules/loss.py | 0 .../src/diffusion_modules/modules/pointnet.py | 0 .../src/diffusion_modules/modules/pvconv.py | 0 .../src/diffusion_modules/modules/se.py | 0 .../diffusion_modules/modules/shared_mlp.py | 0 .../diffusion_modules/modules/voxelization.py | 0 .../src/diffusion_modules/network_ae.py | 0 .../packages/src/diffusion_modules/nn.py | 0 .../src/diffusion_modules/point_voxels.py | 0 .../src/diffusion_modules/resample.py | 0 .../src/diffusion_modules/sparse_network.py | 0 .../packages/src/experiments}/__init__.py | 0 .../src/experiments/utils}/__init__.py | 0 .../src/experiments/utils/wavelet_utils.py | 0 .../packages/src/latent_model}/__init__.py | 0 .../src/latent_model/abstract_volume_nn.py | 0 .../continous_diffusion_interface.py | 0 .../src/latent_model/diffusion_modules/dwt.py | 0 .../diffusion_modules/dwt_utils.py | 0 .../diffusion_modules/fp16_util.py | 0 .../diffusion_modules/gaussian_diffusion.py | 0 .../gaussian_diffusion_wavelet.py | 0 .../src/latent_model/diffusion_modules/nn.py | 0 .../diffusion_modules/resample.py | 0 .../diffusion_modules/sparse_network.py | 0 .../latent_model/diffusion_modules/utils.py | 0 .../packages/src/latent_model/dit_utils.py | 0 .../src/latent_model/gaussian_diffusion.py | 0 .../src/latent_model/latent_dit_utils.py | 0 .../src/latent_model/latent_uvit_utils.py | 0 .../src/latent_model/points_network.py | 0 .../packages/src/latent_model/quantize.py | 0 .../packages/src/latent_model/utils.py | 0 .../src/latent_model/voxels_network.py | 0 .../src/latent_model/wavelet_utils.py | 0 .../src/latent_model/wavelet_vq_model.py | 0 .../packages/src/latent_module.py | 0 .../packages/src/model_utils.py | 0 .../packages/src/mvdream/__init__.py | 0 .../packages/src/mvdream/callbacks.py | 0 .../packages/src/mvdream/camera_utils.py | 0 .../packages/src/mvdream/coco_prompts.py | 0 .../packages/src/mvdream/configs/sd-v1.yaml | 0 .../src/mvdream/configs/sd-v2-base.yaml | 0 .../packages/src/mvdream/constants.py | 0 .../packages/src/mvdream/helper.py | 0 .../packages/src/mvdream/ldm}/__init__.py | 0 .../packages/src/mvdream/ldm/interface.py | 0 .../src/mvdream/ldm/models}/__init__.py | 0 .../src/mvdream/ldm/models/autoencoder.py | 0 .../mvdream/ldm/models/diffusion}/__init__.py | 0 .../src/mvdream/ldm/models/diffusion/ddim.py | 0 .../src/mvdream/ldm/modules}/__init__.py | 0 .../src/mvdream/ldm/modules/attention.py | 0 .../ldm/modules/diffusionmodules}/__init__.py | 0 .../ldm/modules/diffusionmodules/model.py | 0 .../modules/diffusionmodules/openaimodel.py | 0 .../ldm/modules/diffusionmodules/util.py | 0 .../ldm/modules/distributions}/__init__.py | 0 .../modules/distributions/distributions.py | 0 .../packages/src/mvdream/ldm/modules/ema.py | 0 .../mvdream/ldm/modules/encoders}/__init__.py | 0 .../mvdream/ldm/modules/encoders/modules.py | 0 .../packages/src/mvdream/ldm/util.py | 0 .../packages/src/mvdream/model_zoo.py | 0 .../packages/src/mvdream/trainer_old.py | 0 .../packages/src/mvdream/trainermodule.py | 0 .../packages/src/mvdream_module.py | 0 .../packages/src/mvdream_utils.py | 0 .../packages/src/networks/callbacks.py | 0 .../packages/src/utils}/__init__.py | 0 .../packages/src/utils/visualization.py | 0 .../autodesk-wala}/requirements.txt | 0 .../autodesk-wala}/test.py | 0 .../autodesk-wala}/vendor_wala.sh | 0 infrastructure/binocular/README.md | 30 + .../binocular}/config.yaml | 4 + .../binocular/model}/__init__.py | 0 .../binocular}/model/model.py | 0 .../binocular}/packages/config.py | 0 .../docs/audio-transcription/README.md | 0 .../docs/audio-transcription/data_types.py | 0 .../docs/audio-transcription/helpers.py | 0 .../docs/audio-transcription/transcribe.py | 0 .../audio-transcription/whisper_chainlet.py | 0 .../chains-examples}/docs/poems/poems.py | 0 .../custom-engine-builder-control/README.md | 62 ++ .../config.yaml | 1 + .../model/model.py | 0 infrastructure/custom-server/README.md | 37 + .../deepseek-v2-5-instruct-sglang/README.md | 32 + .../deepseek-v2-5-instruct-sglang/config.yaml | 2 + .../infinity-embedding-server/README.md | 33 + .../infinity-embedding-server/config.yaml | 4 + .../llama3-70b-instruct-lmdeploy/README.md | 32 + .../llama3-70b-instruct-lmdeploy/config.yaml | 2 + .../llama3-70b-instruct-sglang/README.md | 32 + .../llama3-70b-instruct-sglang/config.yaml | 2 + .../llama3-8b-instruct-lmdeploy/README.md | 32 + .../llama3-8b-instruct-lmdeploy/config.yaml | 2 + .../llama3-8b-instruct-sglang/README.md | 32 + .../llama3-8b-instruct-sglang/config.yaml | 2 + .../custom-server}/llama3_eval.py | 0 .../custom-server/pixtral-12b/README.md | 54 ++ .../custom-server}/pixtral-12b/config.yaml | 1 + .../pixtral-12b/data/pixtral12b.jinja | 0 .../custom-server/ultravox-0.4/README.md | 50 ++ .../custom-server}/ultravox-0.4/config.yaml | 0 .../custom-server/ultravox-0.5-8b/README.md | 52 ++ .../ultravox-0.5-8b/config.yaml | 0 .../custom-server/ultravox-0.6-70b/README.md | 52 ++ .../ultravox-0.6-70b/config.yaml | 0 .../voxtral-mini-3b-2507/README.md | 46 ++ .../voxtral-mini-3b-2507/config.yaml | 2 +- .../voxtral-small-24b-2507/README.md | 46 ++ .../voxtral-small-24b-2507/config.yaml | 2 +- {grpc => infrastructure/grpc}/Dockerfile | 0 infrastructure/grpc/README.md | 49 ++ {grpc => infrastructure/grpc}/client.py | 0 {grpc => infrastructure/grpc}/config.yaml | 3 + {grpc => infrastructure/grpc}/example.proto | 0 {grpc => infrastructure/grpc}/example_pb2.py | 0 .../grpc}/example_pb2_grpc.py | 0 {grpc => infrastructure/grpc}/model.py | 0 .../grpc}/requirements.txt | 0 infrastructure/jsonformatter/README.md | 29 + .../jsonformatter}/config.yaml | 9 +- .../jsonformatter}/data/schema.json | 0 .../jsonformatter/model}/__init__.py | 0 .../jsonformatter}/model/model.py | 0 infrastructure/layoutlm-document-qa/README.md | 32 + .../layoutlm-document-qa}/config.yaml | 1 + .../layoutlm-document-qa/model}/__init__.py | 0 .../layoutlm-document-qa}/model/model.py | 0 infrastructure/llama-cpp-server/README.md | 56 ++ .../llama-cpp-server}/config.yaml | 1 + .../llama-cpp-server}/cuda.Dockerfile | 0 .../llama-cpp-server}/llama_server_help | 0 .../metrics}/datadog/Dockerfile | 0 infrastructure/metrics/datadog/README.md | 49 ++ .../metrics}/datadog/config.yaml | 1 + .../metrics}/datadog/datadog-vllm-guide.md | 0 .../metrics}/datadog/vllm_conf.yaml | 0 infrastructure/model-cache/README.md | 36 + .../model-cache}/config.yaml | 5 +- .../model-cache}/model/model.py | 0 infrastructure/multiprocessing/README.md | 29 + .../multiprocessing}/config.yaml | 5 +- .../multiprocessing/model}/__init__.py | 0 .../multiprocessing}/model/model.py | 0 .../multiprocessing}/model/test.py | 0 .../ngram-speculator/truss/README.md | 29 + .../ngram-speculator}/truss/config.yaml | 5 +- .../ngram-speculator/truss/model}/__init__.py | 0 .../ngram-speculator}/truss/model/model.py | 0 .../ngram-speculator/trussless/README.md | 47 ++ .../ngram-speculator}/trussless/config.yaml | 1 + .../paddlepaddle/paddleocr-vl/README.md | 52 ++ .../paddlepaddle/paddleocr-vl}/config.yaml | 1 + ip-adapter/README.md | 46 -- kokoro/README.md | 13 - layoutlm-document-qa/README.md | 76 --- llama-cpp-server/README.md | 26 - llama/engine-llama-3-1-70b-instruct/README.md | 5 - llama/engine-llama-3-1-8b-instruct/README.md | 5 - llama/engine-llama-3-3-70b-instruct/README.md | 5 - .../engine-llama-3.1-405b-instruct/README.md | 179 ----- llama/llama-3-70b-instruct/README.md | 63 -- llama/llama-3-8b-instruct/README.md | 63 -- llama/llama-3_1-405b-instruct/README.md | 10 - llama/llama-3_2-11b-vision-instruct/README.md | 72 -- llava/llava-1.6-sgl/README.md | 77 --- llava/llava-v1.5-7b/README.md | 48 -- llava/llava-v1.6-34b/README.md | 48 -- llava/llava-v1.6-34b/config.yaml | 11 - llm/README.md | 33 + llm/cogito/README.md | 46 ++ .../README.md | 57 ++ .../config.yaml | 1 + .../README.md | 58 ++ .../config.yaml | 1 + .../README.md | 58 ++ .../config.yaml | 1 + .../README.md | 58 ++ .../config.yaml | 1 + llm/cogvlm/README.md | 29 + {cogvlm => llm/cogvlm}/config.yaml | 4 + .../cogvlm}/model/__init__.py | 0 {cogvlm => llm/cogvlm}/model/model.py | 0 .../deepseek-ocr}/Bad-Handwriting.png | Bin llm/deepseek/deepseek-ocr/README.md | 54 ++ .../deepseek/deepseek-ocr}/config.yaml | 2 + .../deepseek/deepseek-ocr}/model/__init__.py | 0 .../deepseek/deepseek-ocr}/model/model.py | 0 .../deepseek-ocr}/test_document_ocr.py | 0 .../deepseek-ocr}/visualization_1.png | Bin .../deepseek-ocr}/visualization_2.png | Bin .../deepseek/deepseek-ocr}/visualizer.py | 0 llm/deepseek/deepseek-vl2/README.md | 54 ++ .../deepseek}/deepseek-vl2/config.yaml | 2 + .../README.md | 65 ++ .../config.yaml | 1 + .../README.md | 63 ++ .../config.yaml | 1 + .../README.md | 61 ++ .../config.yaml | 1 + .../README.md | 61 ++ .../config.yaml | 1 + .../README.md | 61 ++ .../config.yaml | 1 + .../falcon3-10B-trt-llm-spec-dec/README.md | 51 ++ .../falcon3-10B-trt-llm-spec-dec/config.yaml | 0 .../README.md | 52 ++ .../config.yaml | 3 + llm/gemma/gemma-2-27b-it-vllm/README.md | 34 + .../gemma}/gemma-2-27b-it-vllm/config.yaml | 1 + .../gemma-2-27b-it-vllm}/model/__init__.py | 0 .../gemma}/gemma-2-27b-it-vllm/model/model.py | 0 llm/gemma/gemma-2-9b-it-vllm/README.md | 34 + .../gemma}/gemma-2-9b-it-vllm/config.yaml | 1 + .../gemma-2-9b-it-vllm/model}/__init__.py | 0 .../gemma}/gemma-2-9b-it-vllm/model/model.py | 0 llm/gemma/gemma-3-27b-it/README.md | 59 ++ .../gemma}/gemma-3-27b-it/config.yaml | 7 +- .../_archive}/llama-2-13b-chat/README.md | 0 .../_archive}/llama-2-13b-chat/config.yaml | 0 .../llama-2-13b-chat/model}/__init__.py | 0 .../_archive}/llama-2-13b-chat/model/model.py | 0 .../llama/_archive}/llama-2-13b/README.md | 0 .../llama/_archive}/llama-2-13b/config.yaml | 0 .../_archive/llama-2-13b/model}/__init__.py | 0 .../_archive}/llama-2-13b/model/model.py | 0 .../_archive}/llama-2-70b-chat/README.md | 0 .../_archive}/llama-2-70b-chat/config.yaml | 0 .../llama-2-70b-chat/model}/__init__.py | 0 .../_archive}/llama-2-70b-chat/model/model.py | 0 .../llama/_archive}/llama-2-70b/README.md | 0 .../llama/_archive}/llama-2-70b/config.yaml | 0 .../_archive/llama-2-70b/model}/__init__.py | 0 .../_archive}/llama-2-70b/model/model.py | 0 .../llama/_archive}/llama-2-7b-chat/README.md | 0 .../_archive}/llama-2-7b-chat/config.yaml | 0 .../llama-2-7b-chat}/model/__init__.py | 0 .../_archive}/llama-2-7b-chat/model/model.py | 0 .../llama/_archive}/llama-2-7b/README.md | 0 .../llama/_archive}/llama-2-7b/config.yaml | 0 .../_archive/llama-2-7b}/model/__init__.py | 0 .../llama/_archive}/llama-2-7b/model/model.py | 0 .../llama-7b-exllama-streaming/README.md | 0 .../llama-7b-exllama-streaming/config.yaml | 0 .../model/__init__.py | 0 .../llama-7b-exllama-streaming/model/model.py | 0 .../_archive}/llama-7b-exllama/README.md | 0 .../_archive}/llama-7b-exllama/config.yaml | 0 .../llama-7b-exllama}/model/__init__.py | 0 .../_archive}/llama-7b-exllama/model/model.py | 0 .../llama/_archive}/llama-7b-vllm/config.yaml | 0 .../_archive/llama-7b-vllm}/model/__init__.py | 0 .../_archive}/llama-7b-vllm/model/model.py | 0 .../llama/_archive}/llama-7b/README.md | 0 .../llama/_archive}/llama-7b/config.yaml | 0 .../llama/_archive}/llama-7b/data/config.json | 0 .../llama-7b/data/generation_config.json | 0 .../data/pytorch_model.bin.index.json | 0 .../_archive/llama-7b}/model/__init__.py | 0 .../llama/_archive}/llama-7b/model/model.py | 0 .../engine-llama-3-1-70b-instruct/README.md | 65 ++ .../engine-llama-3-1-70b-instruct/config.yaml | 1 + .../engine-llama-3-1-8b-instruct/README.md | 63 ++ .../engine-llama-3-1-8b-instruct/config.yaml | 1 + .../engine-llama-3-3-70b-instruct/README.md | 65 ++ .../engine-llama-3-3-70b-instruct/config.yaml | 1 + .../engine-llama-3.1-405b-instruct/README.md | 63 ++ .../config.yaml | 1 + llm/llama/llama-3-1-405b-instruct/README.md | 34 + .../llama-3-1-405b-instruct}/config.yaml | 1 + .../model/__init__.py | 0 .../llama-3-1-405b-instruct}/model/model.py | 0 .../model/sighelper.py | 0 llm/llama/llama-3-1-70b-instruct/README.md | 32 + .../llama/llama-3-1-70b-instruct}/config.yaml | 7 +- .../llama-3-1-70b-instruct}/model/__init__.py | 0 .../llama-3-1-70b-instruct}/model/model.py | 0 .../model/sighelper.py | 0 .../llama-3-1-8b-instruct-sglang/README.md | 34 + .../llama-3-1-8b-instruct-sglang}/config.yaml | 1 + .../model/__init__.py | 0 .../model/model.py | 0 llm/llama/llama-3-1-8b-instruct/README.md | 34 + .../llama/llama-3-1-8b-instruct}/config.yaml | 1 + .../llama-3-1-8b-instruct}/model/__init__.py | 0 .../llama-3-1-8b-instruct}/model/model.py | 0 .../llama-3-2-11b-vision-instruct/README.md | 58 ++ .../config.yaml | 1 + llm/llama/llama-3-70b-instruct/README.md | 32 + .../llama}/llama-3-70b-instruct/config.yaml | 10 +- .../llama-3-70b-instruct}/model/__init__.py | 0 .../llama-3-70b-instruct/model/model.py | 0 llm/llama/llama-3-8b-instruct/README.md | 32 + .../llama}/llama-3-8b-instruct/config.yaml | 10 +- .../llama-3-8b-instruct}/model/__init__.py | 0 .../llama}/llama-3-8b-instruct/model/model.py | 0 .../README.md | 59 ++ .../config.yaml | 3 +- .../data/do.sh | 0 .../README.md | 57 ++ .../config.yaml | 1 + .../data/do.sh | 0 llm/llama/tinyllama-1.1B-chat-v1.0/README.md | 56 ++ .../tinyllama-1.1B-chat-v1.0/config.yaml | 1 + llm/llava/llava-1.6-sgl/README.md | 29 + .../llava}/llava-1.6-sgl/config.yaml | 4 + .../llava/llava-1.6-sgl}/model/__init__.py | 0 .../llava}/llava-1.6-sgl/model/model.py | 0 .../llava}/llava-1.6-sgl/requirements.txt | 0 llm/llava/llava-v1.5-7b/README.md | 29 + .../llava}/llava-v1.5-7b/config.yaml | 4 + .../llava/llava-v1.5-7b}/model/__init__.py | 0 .../llava}/llava-v1.5-7b/model/model.py | 0 .../llava-v1.5-7b/packages/llava/__init__.py | 0 .../llava-v1.5-7b/packages/llava/constants.py | 0 .../packages/llava/conversation.py | 0 .../packages/llava/eval/eval_gpt_review.py | 0 .../llava/eval/eval_gpt_review_bench.py | 0 .../llava/eval/eval_gpt_review_visual.py | 0 .../packages/llava/eval/eval_pope.py | 0 .../packages/llava/eval/eval_science_qa.py | 0 .../llava/eval/eval_science_qa_gpt4.py | 0 .../eval/eval_science_qa_gpt4_requery.py | 0 .../packages/llava/eval/eval_textvqa.py | 0 .../eval/generate_webpage_data_from_table.py | 0 .../packages/llava/eval/m4c_evaluator.py | 0 .../packages/llava/eval/model_qa.py | 0 .../packages/llava/eval/model_vqa.py | 0 .../packages/llava/eval/model_vqa_loader.py | 0 .../packages/llava/eval/model_vqa_mmbench.py | 0 .../packages/llava/eval/model_vqa_qbench.py | 0 .../packages/llava/eval/model_vqa_science.py | 0 .../packages/llava/eval/qa_baseline_gpt35.py | 0 .../packages/llava/eval/run_llava.py | 0 .../llava/eval/summarize_gpt_review.py | 0 .../eval/table/answer/answer_alpaca-13b.jsonl | 0 .../llava/eval/table/answer/answer_bard.jsonl | 0 .../eval/table/answer/answer_gpt35.jsonl | 0 .../eval/table/answer/answer_llama-13b.jsonl | 0 .../eval/table/answer/answer_vicuna-13b.jsonl | 0 .../table/caps_boxes_coco2014_val_80.jsonl | 0 .../packages/llava/eval/table/model.jsonl | 0 .../packages/llava/eval/table/prompt.jsonl | 0 .../packages/llava/eval/table/question.jsonl | 0 .../table/results/test_sqa_llava_13b_v0.json | 0 ...lava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json | 0 .../review/review_alpaca-13b_vicuna-13b.jsonl | 0 .../table/review/review_bard_vicuna-13b.jsonl | 0 .../review/review_gpt35_vicuna-13b.jsonl | 0 .../review/review_llama-13b_vicuna-13b.jsonl | 0 .../packages/llava/eval/table/reviewer.jsonl | 0 .../packages/llava/eval/table/rule.json | 0 .../llava/eval/webpage/figures/alpaca.png | Bin .../llava/eval/webpage/figures/bard.jpg | Bin .../llava/eval/webpage/figures/chatgpt.svg | 0 .../llava/eval/webpage/figures/llama.jpg | Bin .../swords_FILL0_wght300_GRAD0_opsz48.svg | 0 .../llava/eval/webpage/figures/vicuna.jpeg | Bin .../packages/llava/eval/webpage/index.html | 0 .../packages/llava/eval/webpage/script.js | 0 .../packages/llava/eval/webpage/styles.css | 0 .../llava-v1.5-7b/packages/llava/mm_utils.py | 0 .../packages/llava/model/__init__.py | 0 .../packages/llava/model/apply_delta.py | 0 .../packages/llava/model/builder.py | 0 .../packages/llava/model/consolidate.py | 0 .../llava/model/language_model/llava_llama.py | 0 .../llava/model/language_model/llava_mpt.py | 0 .../language_model/mpt/adapt_tokenizer.py | 0 .../model/language_model/mpt/attention.py | 0 .../llava/model/language_model/mpt/blocks.py | 0 .../language_model/mpt/configuration_mpt.py | 0 .../language_model/mpt/custom_embedding.py | 0 .../language_model/mpt/flash_attn_triton.py | 0 .../mpt/hf_prefixlm_converter.py | 0 .../language_model/mpt/meta_init_context.py | 0 .../model/language_model/mpt/modeling_mpt.py | 0 .../llava/model/language_model/mpt/norm.py | 0 .../language_model/mpt/param_init_fns.py | 0 .../packages/llava/model/llava_arch.py | 0 .../packages/llava/model/make_delta.py | 0 .../llava/model/multimodal_encoder/builder.py | 0 .../model/multimodal_encoder/clip_encoder.py | 0 .../model/multimodal_projector/builder.py | 0 .../packages/llava/model/utils.py | 0 .../packages/llava/serve}/__init__.py | 0 .../llava-v1.5-7b/packages/llava/serve/cli.py | 0 .../packages/llava/serve/controller.py | 0 .../llava/serve/examples/extreme_ironing.jpg | Bin .../llava/serve/examples/waterview.jpg | Bin .../packages/llava/serve/gradio_web_server.py | 0 .../packages/llava/serve/model_worker.py | 0 .../packages/llava/serve/register_worker.py | 0 .../packages/llava/serve/test_message.py | 0 .../train/llama_flash_attn_monkey_patch.py | 0 .../train/llama_xformers_attn_monkey_patch.py | 0 .../packages/llava/train/llava_trainer.py | 0 .../packages/llava/train/train.py | 0 .../packages/llava/train/train_mem.py | 0 .../packages/llava/train/train_xformers.py | 0 .../llava-v1.5-7b/packages/llava/utils.py | 0 llm/llava/llava-v1.6-34b/README.md | 29 + llm/llava/llava-v1.6-34b/config.yaml | 15 + .../llava}/llava-v1.6-34b/input.json | 0 .../llava/llava-v1.6-34b}/model/__init__.py | 0 .../llava}/llava-v1.6-34b/model/model.py | 0 {lora => llm/lora}/README.md | 0 llm/lora/engine-lora/README.md | 63 ++ {lora => llm/lora}/engine-lora/config.yaml | 1 + llm/lora/sglang-lora/README.md | 45 ++ {lora => llm/lora}/sglang-lora/config.yaml | 3 +- llm/lora/vllm-lora/README.md | 44 ++ {lora => llm/lora}/vllm-lora/config.yaml | 1 + llm/midnight/README.md | 34 + {midnight => llm/midnight}/config.yaml | 1 + .../midnight}/model/__init__.py | 0 {midnight => llm/midnight}/model/model.py | 0 {midnight => llm/midnight}/requirements.txt | 0 llm/minimax/minimax-m2-1/README.md | 56 ++ .../minimax/minimax-m2-1}/config.yaml | 1 + .../engine-mistral-7b-instruct/README.md | 0 .../engine-mistral-7b-instruct/config.yaml | 0 .../_archive}/mistral-7b-chat/README.md | 0 .../_archive}/mistral-7b-chat/config.yaml | 0 .../mistral-7b-chat}/model/__init__.py | 0 .../_archive}/mistral-7b-chat/model/model.py | 0 .../mistral-7b-instruct-vllm/README.md | 0 .../mistral-7b-instruct-vllm/config.yaml | 0 .../model/__init__.py | 0 .../mistral-7b-instruct-vllm/model/model.py | 0 .../_archive}/mistral-7b-instruct/README.md | 0 .../_archive}/mistral-7b-instruct/config.yaml | 0 .../mistral-7b-instruct}/model/__init__.py | 0 .../mistral-7b-instruct/model/model.py | 0 llm/mistral/engine-devstral/README.md | 60 ++ .../mistral}/engine-devstral/config.yaml | 1 + llm/mistral/engine-mistral-small-3/README.md | 64 ++ .../engine-mistral-small-3/config.yaml | 1 + .../engine-mixtral-8x22b-instruct/README.md | 60 ++ .../engine-mixtral-8x22b-instruct/config.yaml | 1 + .../engine-mixtral-8x7b-instruct/README.md | 64 ++ .../engine-mixtral-8x7b-instruct/config.yaml | 1 + llm/mistral/mistral-7b/README.md | 33 + .../mistral}/mistral-7b/config.yaml | 6 +- .../mistral/mistral-7b/model}/__init__.py | 0 .../mistral}/mistral-7b/model/model.py | 0 llm/mistral/mistral-small-3.1/README.md | 58 ++ .../mistral}/mistral-small-3.1/config.yaml | 9 +- .../README.md | 57 ++ .../TRT-LLM-README.md | 0 .../config.yaml | 3 +- .../data/.gitattributes | 0 .../model}/__init__.py | 0 .../model/model.py | 0 .../packages/client.py | 0 .../ensemble/config.pbtxt | 0 .../postprocessing/1/model.py | 0 .../postprocessing/config.pbtxt | 0 .../preprocessing/1/model.py | 0 .../preprocessing/config.pbtxt | 0 .../tensorrt_llm/config.pbtxt | 0 .../packages/utils.py | 0 llm/mistral/mixtral-8x22b/README.md | 34 + .../mistral}/mixtral-8x22b/config.yaml | 3 +- .../mistral/mixtral-8x22b/model}/__init__.py | 0 .../mistral}/mixtral-8x22b/model/model.py | 0 .../README.md | 55 ++ .../TRT-LLM-README.md | 0 .../config.yaml | 3 +- .../data/.gitattributes | 0 .../model/__init__.py | 0 .../model/model.py | 0 .../packages/client.py | 0 .../ensemble/config.pbtxt | 0 .../postprocessing/1/model.py | 0 .../postprocessing/config.pbtxt | 0 .../preprocessing/1/model.py | 0 .../preprocessing/config.pbtxt | 0 .../tensorrt_llm/config.pbtxt | 0 .../packages/utils.py | 0 .../README.md | 55 ++ .../TRT-LLM-README.md | 0 .../config.yaml | 3 +- .../data/.gitattributes | 0 .../model/__init__.py | 0 .../model/model.py | 0 .../packages/client.py | 0 .../ensemble/config.pbtxt | 0 .../postprocessing/1/model.py | 0 .../postprocessing/config.pbtxt | 0 .../preprocessing/1/model.py | 0 .../preprocessing/config.pbtxt | 0 .../tensorrt_llm/config.pbtxt | 0 .../packages/utils.py | 0 .../README.md | 57 ++ .../TRT-LLM-README.md | 0 .../config.yaml | 2 +- .../data/.gitattributes | 0 .../model/__init__.py | 0 .../model/model.py | 0 .../packages/client.py | 0 .../ensemble/config.pbtxt | 0 .../postprocessing/1/model.py | 0 .../postprocessing/config.pbtxt | 0 .../preprocessing/1/model.py | 0 .../preprocessing/config.pbtxt | 0 .../tensorrt_llm/config.pbtxt | 0 .../packages/utils.py | 0 .../mixtral-8x7b-instruct-trt-llm/README.md | 54 ++ .../TRT-LLM-README.md | 0 .../mixtral-8x7b-instruct-trt-llm/config.yaml | 3 +- .../data/.gitattributes | 0 .../model/__init__.py | 0 .../model/model.py | 0 .../packages/client.py | 0 .../ensemble/config.pbtxt | 0 .../postprocessing/1/model.py | 0 .../postprocessing/config.pbtxt | 0 .../preprocessing/1/model.py | 0 .../preprocessing/config.pbtxt | 0 .../tensorrt_llm/config.pbtxt | 0 .../packages/utils.py | 0 .../README.md | 29 + .../config.yaml | 6 +- .../model/__init__.py | 0 .../model/model.py | 0 .../mixtral-8x7b-instruct-vllm/README.md | 29 + .../mixtral-8x7b-instruct-vllm/config.yaml | 4 + .../model/__init__.py | 0 .../mixtral-8x7b-instruct-vllm/model/model.py | 0 llm/mistral/pixtral-12b/README.md | 53 ++ .../mistral}/pixtral-12b/config.yaml | 1 + .../mistral/pixtral-12b}/model/__init__.py | 0 .../mistral}/pixtral-12b/model/model.py | 0 .../llama-3-1-nemotron-70b-instruct/README.md | 49 ++ .../config.yaml | 3 + .../README.md | 57 ++ .../config.yaml | 11 +- .../README.md | 61 ++ .../base_image/Dockerfile | 0 .../config.yaml | 0 .../model/model.py | 0 llm/nemotron/nemotron-3-nano-nvfp4/README.md | 59 ++ .../nemotron-3-nano-nvfp4}/config.yaml | 1 + llm/nemotron/nemotron-3-nano/README.md | 58 ++ .../nemotron/nemotron-3-nano}/config.yaml | 1 + .../nemotron-nano-12b-v2-vl-bf16/README.md | 57 ++ .../nemotron-nano-12b-v2-vl-bf16}/config.yaml | 11 +- llm/nemotron/nemotron-ultra-253b/README.md | 58 ++ .../nemotron}/nemotron-ultra-253b/config.yaml | 1 + llm/nsql/README.md | 29 + {nsql => llm/nsql}/config.yaml | 4 +- .../nsql}/model/__init__.py | 0 {nsql => llm/nsql}/model/model.py | 0 llm/openai/gpt-oss-120b/README.md | 54 ++ .../openai}/gpt-oss-120b/config.yaml | 1 + llm/openai/gpt-oss-20b/README.md | 54 ++ .../openai}/gpt-oss-20b/config.yaml | 1 + .../personaplex-7b-v1}/Dockerfile | 0 llm/personaplex-7b-v1/README.md | 31 + .../personaplex-7b-v1}/client.py | 0 .../personaplex-7b-v1}/config.yaml | 1 + .../requirements-client.txt | 0 llm/phi/phi-3-mini-128k-instruct/README.md | 29 + llm/phi/phi-3-mini-128k-instruct/config.yaml | 18 + .../model/__init__.py | 0 .../phi-3-mini-128k-instruct/model/model.py | 0 llm/phi/phi-3-mini-4k-instruct/README.md | 29 + llm/phi/phi-3-mini-4k-instruct/config.yaml | 18 + .../phi-3-mini-4k-instruct}/model/__init__.py | 0 .../phi-3-mini-4k-instruct/model/model.py | 0 llm/phi/phi-3.5-mini/README.md | 39 ++ {phi => llm/phi}/phi-3.5-mini/config.yaml | 1 + .../phi/phi-3.5-mini}/model/__init__.py | 0 .../phi/phi-3.5-mini}/model/helper.py | 0 {vllm => llm/phi/phi-3.5-mini}/model/model.py | 0 .../qwen/_archive}/qwen-7b-chat/README.md | 0 .../qwen/_archive}/qwen-7b-chat/config.yaml | 0 .../_archive/qwen-7b-chat}/model/__init__.py | 0 .../_archive}/qwen-7b-chat/model/model.py | 0 .../README.md | 61 ++ .../config.yaml | 1 + .../engine-qwen-2-5-14b-instruct/README.md | 61 ++ .../engine-qwen-2-5-14b-instruct/config.yaml | 1 + .../README.md | 61 ++ .../config.yaml | 1 + .../engine-qwen-2-5-32b-instruct/README.md | 61 ++ .../engine-qwen-2-5-32b-instruct/config.yaml | 1 + .../engine-qwen-2-5-3b-instruct/README.md | 61 ++ .../engine-qwen-2-5-3b-instruct/config.yaml | 1 + .../engine-qwen-2-5-72b-instruct/README.md | 62 ++ .../engine-qwen-2-5-72b-instruct/config.yaml | 1 + .../README.md | 62 ++ .../config.yaml | 1 + .../README.md | 61 ++ .../config.yaml | 1 + .../engine-qwen-2-5-7b-instruct/README.md | 61 ++ .../engine-qwen-2-5-7b-instruct/config.yaml | 1 + .../README.md | 61 ++ .../config.yaml | 1 + llm/qwen/engine-qwen-3-06b/README.md | 59 ++ .../qwen}/engine-qwen-3-06b/config.yaml | 1 + .../README.md | 62 ++ .../config.yaml | 1 + llm/qwen/engine-qwen-3-32b/README.md | 59 ++ .../qwen}/engine-qwen-3-32b/config.yaml | 1 + llm/qwen/engine-qwen-3-4b/README.md | 59 ++ .../qwen}/engine-qwen-3-4b/config.yaml | 1 + {qwen => llm/qwen}/model_auto.py | 0 .../README.md | 53 ++ .../config.yaml | 1 + llm/qwen/qwen-3-235B-sglang/README.md | 56 ++ .../qwen}/qwen-3-235B-sglang/config.yaml | 1 + llm/qwen/qwen-3-30B-A3-coder/README.md | 56 ++ .../qwen}/qwen-3-30B-A3-coder/config.yaml | 1 + llm/qwen/qwen-3-30B-A3-sglang/README.md | 57 ++ .../qwen}/qwen-3-30B-A3-sglang/config.yaml | 1 + llm/qwen/qwen-3-30B-A3-vllm/README.md | 55 ++ .../qwen}/qwen-3-30B-A3-vllm/config.yaml | 1 + llm/qwen/qwen-3-32B-sglang/README.md | 56 ++ .../qwen}/qwen-3-32B-sglang/config.yaml | 1 + .../README.md | 56 ++ .../config.yaml | 1 + .../README.md | 56 ++ .../config.yaml | 1 + llm/qwen/qwen-3-vl-30b-a3b-instruct/README.md | 54 ++ .../qwen-3-vl-30b-a3b-instruct/config.yaml | 2 + llm/qwen/qwen-3-vl-30b-a3b-thinking/README.md | 54 ++ .../qwen-3-vl-30b-a3b-thinking/config.yaml | 2 + llm/qwen/qwen-3-vl-32b/README.md | 57 ++ {qwen => llm/qwen}/qwen-3-vl-32b/config.yaml | 11 +- llm/qwen/qwen-coder-next/README.md | 56 ++ .../qwen}/qwen-coder-next/config.yaml | 2 + llm/qwen/qwen-image/README.md | 37 + .../qwen-image/assets/generated_image.jpg | Bin {qwen => llm/qwen}/qwen-image/config.yaml | 9 +- {qwen => llm/qwen}/qwen-image/model/model.py | 0 llm/qwen/qwen-vl/README.md | 30 + {qwen => llm/qwen}/qwen-vl/config.yaml | 3 + .../qwen/qwen-vl}/model/__init__.py | 0 {qwen => llm/qwen}/qwen-vl/model/model.py | 0 llm/seed/seed-llm/README.md | 56 ++ .../seed/seed-llm}/config.yaml | 1 + llm/z-ai/glm-4-5-air-fp8/README.md | 56 ++ .../z-ai}/glm-4-5-air-fp8/config.yaml | 2 + llm/z-ai/glm-4-5-fp8/README.md | 56 ++ {z-ai => llm/z-ai}/glm-4-5-fp8/config.yaml | 2 + llm/z-ai/glm-4-5-v/README.md | 54 ++ .../z-ai/glm-4-5-v}/config.yaml | 2 + llm/z-ai/glm-4-6-fp8/README.md | 56 ++ {z-ai => llm/z-ai}/glm-4-6-fp8/config.yaml | 2 + llm/z-ai/glm-4-7-flash/README.md | 55 ++ .../z-ai/glm-4-7-flash}/config.yaml | 2 + lora/engine-lora/README.md | 63 -- lora/sglang-lora/README.md | 47 -- lora/vllm-lora/README.md | 46 -- magic-animate/README.md | 89 --- metavoice-1b/README.md | 59 -- midnight/README.md | 254 ------- mistral/engine-mistral-small-3/README.md | 1 - .../engine-mixtral-8x22b-instruct/README.md | 5 - .../engine-mixtral-8x7b-instruct/README.md | 5 - mistral/mistral-7b/README.md | 70 -- .../README.md | 52 -- mistral/mixtral-8x22b/README.md | 78 --- .../README.md | 54 -- .../README.md | 54 -- .../README.md | 54 -- .../mixtral-8x7b-instruct-trt-llm/README.md | 54 -- .../README.md | 62 -- mistral/mixtral-8x7b-instruct-vllm/README.md | 62 -- mistral/pixtral-12b/README.md | 84 --- multiprocessing/README.md | 17 - musicgen-large/README.md | 87 --- musicgen-melody/README.md | 78 --- .../llama-3-1-nemotron-70b-instruct/README.md | 6 - .../README.md | 9 - nsql/README.md | 94 --- openai/gpt-oss-120b/README.md | 72 -- openai/gpt-oss-20b/README.md | 72 -- optimized/README.md | 22 + .../README.md | 56 ++ .../config.yaml | 1 + .../README.md | 56 ++ .../config.yaml | 1 + .../README.md | 56 ++ .../config.yaml | 1 + .../README.md | 56 ++ .../config.yaml | 1 + .../README.md | 56 ++ .../config.yaml | 1 + .../bisv2/nvidia-qwen3-30b-a3b-fp4/README.md | 56 ++ .../nvidia-qwen3-30b-a3b-fp4}/config.yaml | 1 + optimized/bisv2/nvidia-qwen3-8b-fp4/README.md | 56 ++ .../bisv2/nvidia-qwen3-8b-fp4}/config.yaml | 1 + .../README.md | 56 ++ .../config.yaml | 1 + .../qwen-qwen2.5-coder-7b-instruct/README.md | 56 ++ .../config.yaml | 1 + optimized/bisv2/qwen-qwen3-32b-fp4/README.md | 56 ++ .../bisv2/qwen-qwen3-32b-fp4}/config.yaml | 1 + optimized/bisv2/qwen-qwen3-4b-fp8/README.md | 56 ++ .../bisv2/qwen-qwen3-4b-fp8}/config.yaml | 1 + .../README.md | 60 ++ .../config.yaml | 1 + .../README.md | 58 ++ .../config.yaml | 1 + .../briton/google-gemma-3-1b-it/README.md | 59 ++ .../briton/google-gemma-3-1b-it}/config.yaml | 1 + .../briton/google-gemma-3-270m-it/README.md | 59 ++ .../google-gemma-3-270m-it}/config.yaml | 1 + .../README.md | 60 ++ .../config.yaml | 1 + .../briton/google-gemma-3-27b-it/README.md | 59 ++ .../briton/google-gemma-3-27b-it}/config.yaml | 1 + .../meta-llama-llama-3.1-405b-fp8/README.md | 60 ++ .../config.yaml | 1 + .../README.md | 60 ++ .../config.yaml | 1 + .../README.md | 60 ++ .../config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../README.md | 58 ++ .../config.yaml | 1 + .../README.md | 58 ++ .../config.yaml | 1 + .../README.md | 60 ++ .../config.yaml | 1 + .../README.md | 60 ++ .../config.yaml | 1 + .../briton/microsoft-phi-4-fp8/README.md | 60 ++ .../briton/microsoft-phi-4-fp8}/config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../README.md | 58 ++ .../config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../README.md | 62 ++ .../config.yaml | 1 + .../briton/qwen-qwen3-30b-a3b-fp8/README.md | 59 ++ .../qwen-qwen3-30b-a3b-fp8}/config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + optimized/briton/qwen-qwen3-30b-a3b/README.md | 59 ++ .../briton/qwen-qwen3-30b-a3b}/config.yaml | 1 + .../qwen-qwen3-32b-fp4-mlp-only/README.md | 59 ++ .../qwen-qwen3-32b-fp4-mlp-only}/config.yaml | 1 + optimized/briton/qwen-qwen3-32b-fp4/README.md | 59 ++ .../briton/qwen-qwen3-32b-fp4}/config.yaml | 1 + optimized/briton/qwen-qwen3-32b-fp8/README.md | 60 ++ .../briton/qwen-qwen3-32b-fp8}/config.yaml | 1 + .../qwen-qwen3-8b-min-latency-fp8/README.md | 60 ++ .../config.yaml | 1 + .../qwen-qwq-32b-reasoning-fp8/README.md | 58 ++ .../qwen-qwq-32b-reasoning-fp8}/config.yaml | 1 + .../README.md | 59 ++ .../config.yaml | 1 + .../tiiuae-falcon3-10b-instruct-fp8/README.md | 60 ++ .../config.yaml | 1 + orpheus-best-performance/README.md | 8 - personaplex-7b-v1/README.md | 117 ---- phi/phi-3-mini-128k-instruct/README.md | 57 -- phi/phi-3-mini-128k-instruct/config.yaml | 15 - phi/phi-3-mini-4k-instruct/README.md | 57 -- phi/phi-3-mini-4k-instruct/config.yaml | 15 - phi/phi-3.5-mini/README.md | 98 --- piper-tts/README.md | 65 -- playground-v2-aesthetic/README.md | 97 --- poetry.lock | 609 ----------------- pyproject.toml | 21 - .../README.md | 157 ----- .../config.yaml | 30 - .../README.md | 157 ----- .../config.yaml | 31 - .../README.md | 157 ----- .../config.yaml | 30 - .../BEI-qwen-qwen3-embedding-4b-fp8/README.md | 126 ---- .../config.yaml | 28 - .../BEI-qwen-qwen3-embedding-8b-fp8/README.md | 157 ----- .../config.yaml | 29 - .../README.md | 142 ---- .../config.yaml | 31 - qwen/BEI-qwen-qwen3-reranker-4b-fp8/README.md | 126 ---- .../config.yaml | 31 - qwen/BEI-qwen-qwen3-reranker-8b-fp8/README.md | 142 ---- .../config.yaml | 31 - .../README.md | 5 - qwen/engine-qwen-2-5-14b-instruct/README.md | 5 - .../README.md | 5 - qwen/engine-qwen-2-5-32b-instruct/README.md | 5 - qwen/engine-qwen-2-5-3b-instruct/README.md | 5 - qwen/engine-qwen-2-5-72b-instruct/README.md | 7 - .../README.md | 7 - .../README.md | 5 - qwen/engine-qwen-2-5-7b-instruct/README.md | 5 - .../README.md | 5 - qwen/qwen-3-30B-A3-coder/README.md | 162 ----- qwen/qwen-3-30b-omni-thinker/README.md | 256 ------- qwen/qwen-3-30b-omni/README.md | 173 ----- qwen/qwen-3-asr/README.md | 51 -- qwen/qwen-image/README.md | 129 ---- qwen/qwen-vl/README.md | 81 --- segment-anything/README.md | 17 - stable-diffusion/dreamshaper-lcm/README.md | 3 - stable-diffusion/dreamshaper-lcm/config.yaml | 16 - stable-diffusion/playground-v2-trt/README.md | 74 -- .../sd-textual-inversion/README.md | 8 - stable-diffusion/sd-turbo/README.md | 79 --- .../sdxl-controlnet-canny/README.md | 91 --- .../sdxl-controlnet-depth/README.md | 89 --- stable-diffusion/sdxl-controlnet/README.md | 81 --- stable-diffusion/sdxl-lightning/README.md | 87 --- stable-diffusion/sdxl-lora-swapping/README.md | 68 -- stable-diffusion/sdxl-lora/README.md | 65 -- stable-diffusion/sdxl-lora/config.yaml | 20 - stable-diffusion/sdxl-turbo/README.md | 74 -- .../stable-diffusion-3-medium/README.md | 83 --- .../stable-diffusion-inpainting-trt/README.md | 76 --- .../config.yaml | 11 - .../README.md | 74 -- .../stable-diffusion-xl-1.0-trt/README.md | 74 -- .../stable-diffusion-xl-1.0/README.md | 178 ----- stable-diffusion/stable-diffusion/README.md | 83 --- .../stable-video-diffusion/README.md | 90 --- text-embeddings-inference/README.md | 115 ---- tutorials/README.md | 23 + tutorials/cached-weights/README.md | 32 + .../cached-weights}/config.yaml | 1 + .../cached-weights}/doc.yaml | 0 .../cached-weights}/model/__init__.py | 0 .../cached-weights}/model/model.py | 0 .../dynamic-batching}/.gitignore | 0 .../dynamic-batching}/.truss_ignore | 0 tutorials/dynamic-batching/README.md | 33 + .../dynamic-batching}/config.yaml | 3 + .../dynamic-batching}/model/__init__.py | 0 .../dynamic-batching}/model/model.py | 0 .../dynamic-batching/packages}/__init__.py | 0 .../dynamic-batching}/packages/run.py | 0 .../dynamic-batching}/packages/tokenizer.py | 0 .../packages/whisper_utils.py | 0 .../dynamic-batching}/test.py | 0 tutorials/getting-started-bert/README.md | 31 + .../getting-started-bert}/config.yaml | 2 + .../getting-started-bert}/doc.yaml | 0 .../getting-started-bert}/model/__init__.py | 0 .../getting-started-bert}/model/model.py | 0 tutorials/image-generation/README.md | 32 + .../image-generation}/config.yaml | 2 + .../image-generation}/doc.yaml | 0 .../image-generation}/model/__init__.py | 0 .../image-generation}/model/model.py | 0 tutorials/llm-basics/README.md | 33 + {02-llm => tutorials/llm-basics}/config.yaml | 2 + {02-llm => tutorials/llm-basics}/doc.yaml | 0 .../llm-basics}/model/__init__.py | 0 .../llm-basics}/model/model.py | 0 tutorials/llm-streaming/README.md | 30 + .../llm-streaming}/config.yaml | 2 + .../llm-streaming}/doc.yaml | 0 .../llm-streaming}/model/__init__.py | 0 .../llm-streaming}/model/model.py | 0 tutorials/private-huggingface/README.md | 31 + .../private-huggingface}/config.yaml | 4 + .../private-huggingface}/doc.yaml | 0 .../private-huggingface}/model/__init__.py | 0 .../private-huggingface}/model/model.py | 0 tutorials/speech-to-text/README.md | 31 + .../speech-to-text}/config.yaml | 2 + .../speech-to-text}/model/__init__.py | 0 .../speech-to-text}/model/model.py | 0 tutorials/system-packages/README.md | 32 + .../system-packages}/config.yaml | 2 + .../system-packages}/doc.yaml | 0 .../system-packages/model/__init__.py | 0 .../system-packages}/model/model.py | 0 ultravox/README.md | 97 --- whisper/faster-whisper-small/README.md | 69 -- whisper/whisper-streaming/README.md | 62 -- whisper/whisper-truss/README.md | 79 --- whisper/whisper-v3-truss-base64/README.md | 75 -- whisper/whisper-v3-truss/README.md | 72 -- whisper/whisperx-truss/README.md | 84 --- xtts-streaming/README.md | 115 ---- xtts-v2-truss/README.md | 73 -- z-ai/glm-4-5-air-fp8/README.md | 42 -- z-ai/glm-4-5-fp8/README.md | 42 -- 2215 files changed, 16934 insertions(+), 31419 deletions(-) create mode 100644 .claude/CLAUDE.md delete mode 100644 .droid.yaml delete mode 100644 .github/actions/setup-python/action.yml delete mode 100644 .isort.cfg delete mode 100644 .tool-versions delete mode 100644 07-high-performance-dynamic-batching/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding/README.md delete mode 100644 11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2/README.md create mode 100644 _archive/README.md rename {deepspeed-mii => _archive/deepspeed-mii}/README.md (100%) rename {deepspeed-mii => _archive/deepspeed-mii}/config.yaml (100%) rename {01-getting-started-bert => _archive/deepspeed-mii}/model/__init__.py (100%) rename {deepspeed-mii => _archive/deepspeed-mii}/model/model.py (100%) rename {model_cach_gcs => _archive/model_cach_gcs}/config.yaml (100%) rename {model_cach_gcs => _archive/model_cach_gcs}/model/model.py (100%) rename {nous-capybara => _archive/nous-capybara}/nous-capybara-34b-openai/README.md (100%) rename {nous-capybara => _archive/nous-capybara}/nous-capybara-34b-openai/config.yaml (100%) rename {02-llm => _archive/nous-capybara/nous-capybara-34b-openai}/model/__init__.py (100%) rename {nous-capybara => _archive/nous-capybara}/nous-capybara-34b-openai/model/model.py (100%) rename {nous-capybara => _archive/nous-capybara}/nous-capybara-34b/README.md (100%) rename {nous-capybara => _archive/nous-capybara}/nous-capybara-34b/config.yaml (100%) rename {03-llm-with-streaming => _archive/nous-capybara/nous-capybara-34b}/model/__init__.py (100%) rename {nous-capybara => _archive/nous-capybara}/nous-capybara-34b/model/model.py (100%) rename {vllm => _archive/vllm}/README.md (100%) rename {vllm => _archive/vllm}/config.yaml (100%) rename {04-image-generation => _archive/vllm}/model/__init__.py (100%) rename {phi/phi-3.5-mini => _archive/vllm}/model/helper.py (100%) rename {phi/phi-3.5-mini => _archive/vllm}/model/model.py (100%) rename {whisper => _archive}/whisper-torchserve/README.md (100%) rename {whisper => _archive}/whisper-torchserve/config.yaml (100%) rename {whisper => _archive}/whisper-torchserve/data/config.properties (100%) rename {05-speech-to-text => _archive/whisper-torchserve}/model/__init__.py (100%) rename {whisper => _archive}/whisper-torchserve/model/model.py (100%) rename 11-embeddings-reranker-classification-tensorrt/README.md => _internal/11-trt-original-README.md (100%) create mode 100644 _internal/README.md rename {assets => _internal/assets}/comfyui-screenshot-1.png (100%) rename {assets => _internal/assets}/comfyui-screenshot-2.png (100%) rename {assets => _internal/assets}/comfyui-screenshot-3.png (100%) rename {assets => _internal/assets}/sdxl-controlnet-workflow.json (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/deepseek-v3-0324/README.md (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/deepseek-v3-0324/config.yaml (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/glm47/config.yaml (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/gpt-oss-120b/README.md (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/gpt-oss-120b/config.yaml (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/kimi-k2-instruct/README.md (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/kimi-k2-instruct/config.yaml (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/llama-4-maverick/config.yaml (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/qwen3-Coder-30b-instruct/README.md (100%) rename {baseten-inference-stack-v2-templates => _internal/baseten-inference-stack-v2-templates}/qwen3-Coder-30b-instruct/config.yaml (100%) create mode 100644 _internal/bin/generate_readmes.py rename {bin => _internal/bin}/image.txt (100%) create mode 100644 _internal/bin/test_all.py rename {bin => _internal/bin}/test_example.py (100%) rename {bin => _internal/bin}/test_truss_deploy.py (98%) create mode 100644 _internal/bin/validate_all.py rename {bin => _internal/bin}/validate_ci.py (100%) rename {dockerfiles => _internal/dockerfiles}/ComfyUI.dockerfile (100%) rename {essential => _internal/essential}/config.yaml (100%) rename {internal => _internal/internal}/config.yaml (100%) rename {templates => _internal/templates}/README.md (100%) rename {templates => _internal/templates}/faster-whisper-truss/config.yaml (100%) rename {06-high-performance-cached-weights => _internal/templates/faster-whisper-truss}/model/__init__.py (100%) rename {templates => _internal/templates}/faster-whisper-truss/model/model.py (100%) rename {templates => _internal/templates}/generate.py (100%) rename {templates => _internal/templates}/generate.yaml (100%) rename {templates => _internal/templates}/transformers-openai-compatible/config.yaml (100%) rename {07-high-performance-dynamic-batching => _internal/templates/transformers-openai-compatible}/model/__init__.py (100%) rename {templates => _internal/templates}/transformers-openai-compatible/model/model.py (100%) rename {mistral/mixtral-8x22b-trt-int8-weights-only => _internal/templates/trt-llm}/TRT-LLM-README.md (100%) rename {templates => _internal/templates}/trt-llm/config.yaml (100%) rename {mistral/mixtral-8x22b-trt-int8-weights-only => _internal/templates/trt-llm}/data/.gitattributes (100%) rename {07-high-performance-dynamic-batching/packages => _internal/templates/trt-llm/model}/__init__.py (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => _internal/templates/trt-llm}/model/model.py (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => _internal/templates/trt-llm}/packages/client.py (100%) rename {templates => _internal/templates}/trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt.jinja (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => _internal/templates/trt-llm}/packages/inflight_batcher_llm/postprocessing/1/model.py (100%) rename {templates => _internal/templates}/trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt.jinja (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => _internal/templates/trt-llm}/packages/inflight_batcher_llm/preprocessing/1/model.py (100%) rename {templates => _internal/templates}/trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt.jinja (100%) rename {mistral/mixtral-8x22b-trt-int8-weights-only => _internal/templates/trt-llm}/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt (100%) rename {mistral/mixtral-8x22b-trt-int8-weights-only => _internal/templates/trt-llm}/packages/utils.py (100%) rename {11-embeddings-reranker-classification-tensorrt => _internal}/templating/.internal_tei/Dockerfile (100%) rename {11-embeddings-reranker-classification-tensorrt => _internal}/templating/.internal_tei/roll_out_docker.sh (100%) rename {11-embeddings-reranker-classification-tensorrt => _internal}/templating/README.md (100%) rename {11-embeddings-reranker-classification-tensorrt => _internal}/templating/deploy_all.py (100%) rename {11-embeddings-reranker-classification-tensorrt => _internal}/templating/generate_templates.py (98%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-70b-instruct/high_throughput/README.md (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-70b-instruct/high_throughput/config.yaml (100%) rename {09-private-huggingface => _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput}/model/__init__.py (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-70b-instruct/large_context/README.md (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-70b-instruct/large_context/config.yaml (100%) rename {10-using-system-packages => _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context}/model/__init__.py (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-70b-instruct/low_ttft/README.md (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-70b-instruct/low_ttft/config.yaml (100%) rename {audiogen-medium => _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft}/model/__init__.py (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-8b-instruct/high_throughput/README.md (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-8b-instruct/high_throughput/config.yaml (100%) rename {autodesk-wala => _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput}/model/__init__.py (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-8b-instruct/large_context/README.md (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-8b-instruct/large_context/config.yaml (100%) rename {autodesk-wala/packages/src => _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/model}/__init__.py (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-8b-instruct/low_ttft/README.md (100%) rename {trt-llm-engine-builder-templates => _internal/trt-llm-engine-builder-templates}/llama-3_1-8b-instruct/low_ttft/config.yaml (100%) rename {autodesk-wala/packages/src/diffusion_modules => _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/model}/__init__.py (100%) create mode 100644 audio/README.md create mode 100644 audio/audiogen-medium/README.md rename {audiogen-medium => audio/audiogen-medium}/config.yaml (92%) rename {autodesk-wala/packages/src/experiments => audio/audiogen-medium/model}/__init__.py (100%) rename {audiogen-medium => audio/audiogen-medium}/model/model.py (100%) create mode 100644 audio/chatterbox-tts/README.md rename {chatterbox-tts => audio/chatterbox-tts}/config.yaml (59%) rename {chatterbox-tts => audio/chatterbox-tts}/docker/Dockerfile (100%) rename {chatterbox-tts => audio/chatterbox-tts}/docker/docker_build.sh (100%) rename {chatterbox-tts => audio/chatterbox-tts}/input/obama_8s.wav (100%) rename {autodesk-wala/packages/src/experiments/utils => audio/chatterbox-tts/model}/__init__.py (100%) rename {chatterbox-tts => audio/chatterbox-tts}/model/model.py (100%) rename {chatterbox-tts => audio/chatterbox-tts}/run_tts.py (100%) create mode 100644 audio/kokoro/README.md rename {kokoro => audio/kokoro}/call.py (100%) rename {kokoro => audio/kokoro}/config.yaml (90%) rename {autodesk-wala/packages/src/latent_model => audio/kokoro/model}/__init__.py (100%) rename {kokoro => audio/kokoro}/model/model.py (100%) create mode 100644 audio/metavoice-1b/README.md rename {metavoice-1b => audio/metavoice-1b}/config.yaml (94%) rename {metavoice-1b => audio/metavoice-1b}/data/bria.mp3 (100%) rename {autodesk-wala/packages/src/mvdream/ldm => audio/metavoice-1b/model}/__init__.py (100%) rename {metavoice-1b => audio/metavoice-1b}/model/model.py (100%) rename {metavoice-1b => audio/metavoice-1b}/process.py (100%) rename {metavoice-1b => audio/metavoice-1b}/requirements.txt (100%) create mode 100644 audio/musicgen-large/README.md rename {musicgen-large => audio/musicgen-large}/config.yaml (92%) rename {autodesk-wala/packages/src/mvdream/ldm/models => audio/musicgen-large/model}/__init__.py (100%) rename {musicgen-large => audio/musicgen-large}/model/model.py (100%) create mode 100644 audio/musicgen-melody/README.md rename {musicgen-melody => audio/musicgen-melody}/config.yaml (90%) rename {autodesk-wala/packages/src/mvdream/ldm/models/diffusion => audio/musicgen-melody/model}/__init__.py (100%) rename {musicgen-melody => audio/musicgen-melody}/model/model.py (100%) create mode 100644 audio/nvidia-parakeet/README.md rename {nvidia/parakeet-tdt-0_6b-v2 => audio/nvidia-parakeet}/config.yaml (93%) rename {autodesk-wala/packages/src/mvdream/ldm/modules => audio/nvidia-parakeet/model}/__init__.py (100%) rename {nvidia/parakeet-tdt-0_6b-v2 => audio/nvidia-parakeet}/model/model.py (100%) create mode 100644 audio/orpheus-3b-websockets/README.md rename {orpheus-3b-websockets => audio/orpheus-3b-websockets}/call.py (100%) rename {orpheus-3b-websockets => audio/orpheus-3b-websockets}/config.yaml (93%) rename {orpheus-3b-websockets => audio/orpheus-3b-websockets}/model/model.py (100%) rename {orpheus-3b-websockets => audio/orpheus-3b-websockets}/snac_batching_quantization_dev.py (100%) create mode 100644 audio/orpheus-best-performance/README.md rename {orpheus-best-performance => audio/orpheus-best-performance}/call.py (100%) rename {orpheus-best-performance => audio/orpheus-best-performance}/config.yaml (96%) rename {orpheus-best-performance => audio/orpheus-best-performance}/model/model.py (100%) rename {orpheus-best-performance => audio/orpheus-best-performance}/snac_batching_quantization_dev.py (100%) create mode 100644 audio/piper-tts/README.md rename {piper-tts => audio/piper-tts}/config.yaml (89%) rename {autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules => audio/piper-tts/model}/__init__.py (100%) rename {piper-tts => audio/piper-tts}/model/model.py (100%) create mode 100644 audio/qwen-asr/README.md rename {qwen/qwen-3-asr => audio/qwen-asr}/config.yaml (84%) create mode 100644 audio/qwen-omni-thinker/README.md rename {qwen/qwen-3-30b-omni-thinker => audio/qwen-omni-thinker}/config.yaml (90%) create mode 100644 audio/qwen-omni/README.md rename {qwen/qwen-3-30b-omni => audio/qwen-omni}/config.yaml (82%) rename {qwen/qwen-3-30b-omni => audio/qwen-omni}/model/model.py (100%) create mode 100644 audio/sesame-csm-1b/README.md rename {sesame-csm-1b => audio/sesame-csm-1b}/config.yaml (71%) rename {autodesk-wala/packages/src/mvdream/ldm/modules/distributions => audio/sesame-csm-1b/model}/__init__.py (100%) rename {sesame-csm-1b => audio/sesame-csm-1b}/model/model.py (100%) create mode 100644 audio/ultravox/README.md rename {ultravox => audio/ultravox}/config.yaml (61%) rename {autodesk-wala/packages/src/mvdream/ldm/modules/encoders => audio/ultravox/model}/__init__.py (100%) rename {ultravox => audio/ultravox}/model/model.py (100%) create mode 100644 audio/voxtral-streaming-4b/README.md rename {mistral => audio}/voxtral-streaming-4b/config.yaml (69%) rename {mistral => audio}/voxtral-streaming-4b/streaming.py (100%) create mode 100644 audio/whisper/faster-whisper-small/README.md rename {whisper => audio/whisper}/faster-whisper-small/config.yaml (100%) rename {autodesk-wala/packages/src/utils => audio/whisper/faster-whisper-small/model}/__init__.py (100%) rename {whisper => audio/whisper}/faster-whisper-small/model/model.py (100%) create mode 100644 audio/whisper/faster-whisper-v2/README.md rename {whisper => audio/whisper}/faster-whisper-v2/config.yaml (100%) rename {binocular => audio/whisper/faster-whisper-v2}/model/__init__.py (100%) rename {whisper => audio/whisper}/faster-whisper-v2/model/model.py (100%) create mode 100644 audio/whisper/faster-whisper-v3/README.md rename {whisper => audio/whisper}/faster-whisper-v3/config.yaml (100%) rename {chatterbox-tts => audio/whisper/faster-whisper-v3}/model/__init__.py (100%) rename {whisper => audio/whisper}/faster-whisper-v3/model/model.py (100%) rename {gfp-gan => audio/whisper/whisper-openai}/LICENSE (100%) create mode 100644 audio/whisper/whisper-openai/README.md rename {whisper/whisper-truss => audio/whisper/whisper-openai}/config.yaml (95%) rename {gfp-gan => audio/whisper/whisper-openai}/data/.gitkeep (100%) rename {clip => audio/whisper/whisper-openai}/model/__init__.py (100%) rename {whisper/whisper-truss => audio/whisper/whisper-openai}/model/model.py (100%) create mode 100644 audio/whisper/whisper-streaming/README.md rename {whisper => audio/whisper}/whisper-streaming/config.yaml (67%) rename {cogvlm => audio/whisper/whisper-streaming}/model/__init__.py (100%) rename {whisper => audio/whisper}/whisper-streaming/model/model.py (100%) rename {whisper => audio/whisper}/whisper-streaming/packages/whisper_streaming/line_packet.py (100%) rename {whisper => audio/whisper}/whisper-streaming/packages/whisper_streaming/whisper_online.py (100%) rename {whisper => audio/whisper}/whisper-streaming/packages/whisper_streaming/whisper_online_server.py (100%) rename {whisper => audio/whisper}/whisper-streaming/requirements.txt (100%) create mode 100644 audio/whisper/whisper-v3-base64/README.md rename {whisper/whisper-v3-truss-base64 => audio/whisper/whisper-v3-base64}/config.yaml (95%) rename {comfyui-truss => audio/whisper/whisper-v3-base64}/model/__init__.py (100%) rename {whisper/whisper-v3-truss-base64 => audio/whisper/whisper-v3-base64}/model/model.py (100%) create mode 100644 audio/whisper/whisper-v3/README.md rename {whisper/whisper-v3-truss => audio/whisper/whisper-v3}/config.yaml (95%) rename {control-net-qrcode => audio/whisper/whisper-v3}/model/__init__.py (100%) rename {whisper/whisper-v3-truss => audio/whisper/whisper-v3}/model/model.py (100%) create mode 100644 audio/whisper/whisperx/README.md rename {whisper/whisperx-truss => audio/whisper/whisperx}/config.yaml (90%) rename {deepfloyd-xl => audio/whisper/whisperx}/model/__init__.py (100%) rename {whisper/whisperx-truss => audio/whisper/whisperx}/model/model.py (100%) create mode 100644 audio/xtts-streaming/README.md rename {xtts-streaming => audio/xtts-streaming}/config.yaml (61%) rename {deepspeed-mii => audio/xtts-streaming}/model/__init__.py (100%) rename {xtts-streaming => audio/xtts-streaming}/model/model.py (100%) rename {xtts-streaming => audio/xtts-streaming}/requirements.txt (100%) create mode 100644 audio/xtts-v2/README.md rename {xtts-v2-truss => audio/xtts-v2}/config.yaml (90%) rename {dis-segmentation => audio/xtts-v2}/model/__init__.py (100%) rename {xtts-v2-truss => audio/xtts-v2}/model/model.py (100%) delete mode 100644 audiogen-medium/README.md delete mode 100644 autodesk-wala/README.md delete mode 100644 chatterbox-tts/README.md delete mode 100644 cogito/README.md delete mode 100644 cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md delete mode 100644 cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md delete mode 100644 cogito/cogito-v2-preview-llama-405B-vllm/README.md delete mode 100644 cogito/cogito-v2-preview-llama-70B-vllm/README.md delete mode 100644 cogvlm/README.md delete mode 100644 comfyui-truss/README.md delete mode 100644 control-net-qrcode/README.md delete mode 100644 custom-engine-builder-control/README.md delete mode 100644 custom-server/README.md delete mode 100644 custom-server/infinity-embedding-server/README.md delete mode 100644 custom-server/pixtral-12b/README.md delete mode 100644 custom-server/ultravox-0.4/README.md delete mode 100644 custom-server/ultravox-0.5-8b/README.md delete mode 100644 custom-server/ultravox-0.6-70b/README.md delete mode 100644 custom-server/voxtral-mini-3b-2507/README.md delete mode 100644 custom-server/voxtral-small-24b-2507/README.md delete mode 100644 deepfloyd-xl/README.md delete mode 100644 deepseek-ocr/README.md delete mode 100644 deepseek/engine-deepseek-r1-distill-llama-70b/README.md delete mode 100644 deepseek/engine-deepseek-r1-distill-llama-8b/README.md delete mode 100644 deepseek/engine-deepseek-r1-distill-qwen-14b/README.md delete mode 100644 deepseek/engine-deepseek-r1-distill-qwen-32b/README.md delete mode 100644 deepseek/engine-deepseek-r1-distill-qwen-7b/README.md delete mode 100644 dis-segmentation/README.md create mode 100644 embeddings/README.md create mode 100644 embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding => embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding}/config.yaml (83%) create mode 100644 embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding => embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding}/config.yaml (83%) create mode 100644 embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding => embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding}/config.yaml (83%) create mode 100644 embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base => embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base}/config.yaml (85%) create mode 100644 embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8 => embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8}/config.yaml (86%) create mode 100644 embeddings/bei/baai-bge-en-icl-embedding-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8 => embeddings/bei/baai-bge-en-icl-embedding-fp8}/config.yaml (86%) create mode 100644 embeddings/bei/baai-bge-large-en-v1.5-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding => embeddings/bei/baai-bge-large-en-v1.5-embedding}/config.yaml (84%) create mode 100644 embeddings/bei/baai-bge-m3-embedding-dense/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense => embeddings/bei/baai-bge-m3-embedding-dense}/config.yaml (85%) create mode 100644 embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding => embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding}/config.yaml (84%) create mode 100644 embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large/README.md rename {11-embeddings-reranker-classification-tensorrt => embeddings/bei/baai-bge-reranker-large}/BEI-Bert-baai-bge-reranker-large/config.yaml (86%) create mode 100644 embeddings/bei/baai-bge-reranker-large/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large => embeddings/bei/baai-bge-reranker-large}/config.yaml (86%) create mode 100644 embeddings/bei/baai-bge-reranker-v2-m3-multilingual/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual => embeddings/bei/baai-bge-reranker-v2-m3-multilingual}/config.yaml (86%) create mode 100644 embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8 => embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8}/config.yaml (84%) create mode 100644 embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8 => embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8}/config.yaml (85%) create mode 100644 embeddings/bei/google-embeddinggemma-300m/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m => embeddings/bei/google-embeddinggemma-300m}/config.yaml (84%) create mode 100644 embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8 => embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8}/config.yaml (84%) create mode 100644 embeddings/bei/intfloat-multilingual-e5-large-instruct/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct => embeddings/bei/intfloat-multilingual-e5-large-instruct}/config.yaml (83%) create mode 100644 embeddings/bei/jina-ai-jina-embeddings-v2-base-en/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en => embeddings/bei/jina-ai-jina-embeddings-v2-base-en}/config.yaml (83%) create mode 100644 embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8 => embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8}/config.yaml (84%) create mode 100644 embeddings/bei/jinaai-jina-embeddings-v2-base-code/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code => embeddings/bei/jinaai-jina-embeddings-v2-base-code}/config.yaml (83%) create mode 100644 embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt => embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding}/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml (83%) create mode 100644 embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding => embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding}/config.yaml (83%) create mode 100644 embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8 => embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8}/config.yaml (86%) create mode 100644 embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8 => embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8}/config.yaml (86%) create mode 100644 embeddings/bei/ncbi-medcpt-cross-encoder-reranker/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker => embeddings/bei/ncbi-medcpt-cross-encoder-reranker}/config.yaml (86%) create mode 100644 embeddings/bei/ner-bert-base-ner-uncased/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased => embeddings/bei/ner-bert-base-ner-uncased}/config.yaml (87%) create mode 100644 embeddings/bei/nomic-ai-nomic-embed-code-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8 => embeddings/bei/nomic-ai-nomic-embed-code-fp8}/config.yaml (85%) create mode 100644 embeddings/bei/nomic-ai-nomic-embed-text-v1.5/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5 => embeddings/bei/nomic-ai-nomic-embed-text-v1.5}/config.yaml (83%) create mode 100644 embeddings/bei/nomic-ai-nomic-embed-text-v2-moe/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe => embeddings/bei/nomic-ai-nomic-embed-text-v2-moe}/config.yaml (83%) create mode 100644 embeddings/bei/nvidia-llama-embed-nemotron-8b/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b => embeddings/bei/nvidia-llama-embed-nemotron-8b}/config.yaml (83%) create mode 100644 embeddings/bei/nvidia-llama-nemotron-embed-1b-v2/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2 => embeddings/bei/nvidia-llama-nemotron-embed-1b-v2}/config.yaml (83%) create mode 100644 embeddings/bei/papluca-xlm-roberta-base-language-detection-classification/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification => embeddings/bei/papluca-xlm-roberta-base-language-detection-classification}/config.yaml (84%) create mode 100644 embeddings/bei/qwen-qwen3-embedding-0.6b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8 => embeddings/bei/qwen-qwen3-embedding-0.6b-fp8}/config.yaml (84%) create mode 100644 embeddings/bei/qwen-qwen3-embedding-4b-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4 => embeddings/bei/qwen-qwen3-embedding-4b-fp4}/config.yaml (84%) create mode 100644 embeddings/bei/qwen-qwen3-embedding-4b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8 => embeddings/bei/qwen-qwen3-embedding-4b-fp8}/config.yaml (84%) create mode 100644 embeddings/bei/qwen-qwen3-embedding-8b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8 => embeddings/bei/qwen-qwen3-embedding-8b-fp8}/config.yaml (84%) create mode 100644 embeddings/bei/qwen-qwen3-reranker-0.6b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8 => embeddings/bei/qwen-qwen3-reranker-0.6b-fp8}/config.yaml (85%) create mode 100644 embeddings/bei/qwen-qwen3-reranker-4b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8 => embeddings/bei/qwen-qwen3-reranker-4b-fp8}/config.yaml (86%) create mode 100644 embeddings/bei/qwen-qwen3-reranker-8b-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4 => embeddings/bei/qwen-qwen3-reranker-8b-fp4}/config.yaml (86%) create mode 100644 embeddings/bei/qwen-qwen3-reranker-8b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8 => embeddings/bei/qwen-qwen3-reranker-8b-fp8}/config.yaml (86%) create mode 100644 embeddings/bei/redis-langcache-embed-v2/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2 => embeddings/bei/redis-langcache-embed-v2}/config.yaml (84%) create mode 100644 embeddings/bei/salesforce-sfr-embedding-mistral-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8 => embeddings/bei/salesforce-sfr-embedding-mistral-fp8}/config.yaml (84%) create mode 100644 embeddings/bei/samlowe-roberta-base-go_emotions-classification/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification => embeddings/bei/samlowe-roberta-base-go_emotions-classification}/config.yaml (85%) create mode 100644 embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding => embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding}/config.yaml (83%) create mode 100644 embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8 => embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8}/config.yaml (85%) create mode 100644 embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0 => embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0}/config.yaml (82%) create mode 100644 embeddings/bei/tanaos-tanaos-ner-v1/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1 => embeddings/bei/tanaos-tanaos-ner-v1}/config.yaml (87%) create mode 100644 embeddings/bei/taylorai-bge-micro-v2/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2 => embeddings/bei/taylorai-bge-micro-v2}/config.yaml (84%) create mode 100644 embeddings/bei/voyageai-voyage-4-nano/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano => embeddings/bei/voyageai-voyage-4-nano}/config.yaml (84%) create mode 100644 embeddings/bei/whereisai-uae-large-v1-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding => embeddings/bei/whereisai-uae-large-v1-embedding}/config.yaml (84%) create mode 100644 embeddings/clip/README.md rename {clip => embeddings/clip}/config.yaml (76%) rename {flux.1-dev-trt-b200 => embeddings/clip}/model/__init__.py (100%) rename {clip => embeddings/clip}/model/model.py (100%) create mode 100644 embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding => embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding}/config.yaml (94%) create mode 100644 embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding => embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding}/config.yaml (94%) create mode 100644 embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding => embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding}/config.yaml (94%) create mode 100644 embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base => embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base}/config.yaml (94%) create mode 100644 embeddings/tei/baai-bge-reranker-large/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large => embeddings/tei/baai-bge-reranker-large}/config.yaml (95%) create mode 100644 embeddings/tei/google-embeddinggemma-300m/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m => embeddings/tei/google-embeddinggemma-300m}/config.yaml (94%) create mode 100644 embeddings/tei/intfloat-multilingual-e5-large-instruct/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct => embeddings/tei/intfloat-multilingual-e5-large-instruct}/config.yaml (93%) create mode 100644 embeddings/tei/jina-ai-jina-embeddings-v2-base-en/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en => embeddings/tei/jina-ai-jina-embeddings-v2-base-en}/config.yaml (94%) create mode 100644 embeddings/tei/jinaai-jina-embeddings-v2-base-code/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code => embeddings/tei/jinaai-jina-embeddings-v2-base-code}/config.yaml (94%) create mode 100644 embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding => embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding}/config.yaml (94%) create mode 100644 embeddings/tei/nomic-ai-nomic-embed-text-v1.5/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5 => embeddings/tei/nomic-ai-nomic-embed-text-v1.5}/config.yaml (94%) create mode 100644 embeddings/tei/nomic-ai-nomic-embed-text-v2-moe/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe => embeddings/tei/nomic-ai-nomic-embed-text-v2-moe}/config.yaml (94%) create mode 100644 embeddings/tei/redis-langcache-embed-v2/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2 => embeddings/tei/redis-langcache-embed-v2}/config.yaml (95%) create mode 100644 embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding => embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding}/config.yaml (93%) create mode 100644 embeddings/tei/taylorai-bge-micro-v2/README.md rename {11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2 => embeddings/tei/taylorai-bge-micro-v2}/config.yaml (95%) create mode 100644 embeddings/text-embeddings-inference/README.md rename {text-embeddings-inference => embeddings/text-embeddings-inference}/config.yaml (89%) delete mode 100644 falcon/falcon3-10B-trt-llm-spec-dec/README.md delete mode 100644 falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md delete mode 100644 flux.1-dev-trt-b200/README.md delete mode 100644 flux/README.md delete mode 100644 flux/schnell/README.md delete mode 100644 fotographer/zenctrl/README.md delete mode 100644 gemma/gemma-2-27b-it-vllm/README.md delete mode 100644 gemma/gemma-2-9b-it-vllm/README.md delete mode 100644 gfp-gan/README.md delete mode 100644 grpc/README.md delete mode 100644 image-segmentation/config.yaml create mode 100644 image/README.md create mode 100644 image/comfyui/README.md rename {comfyui-truss => image/comfyui}/config.yaml (89%) rename {comfyui-truss => image/comfyui}/data/comfy_ui_workflow.json (100%) rename {comfyui-truss => image/comfyui}/data/model.json (100%) rename {comfyui-truss => image/comfyui}/examples/animate-diff/model.json (100%) rename {comfyui-truss => image/comfyui}/examples/animate-diff/workflow.json (100%) create mode 100644 image/comfyui/examples/anime-style-transfer/README.md rename {comfyui-truss => image/comfyui}/examples/anime-style-transfer/config.yaml (86%) rename {comfyui-truss => image/comfyui}/examples/anime-style-transfer/workflow.json (100%) rename {comfyui-truss => image/comfyui}/examples/sdxl-controlnet/model.json (100%) rename {comfyui-truss => image/comfyui}/examples/sdxl-controlnet/workflow.json (100%) rename {comfyui-truss => image/comfyui}/examples/sdxl-with-refiner/model.json (100%) rename {comfyui-truss => image/comfyui}/examples/sdxl-with-refiner/workflow.json (100%) rename {flux.1-dev-trt-b200/model/demo_diffusion => image/comfyui/model}/__init__.py (100%) rename {comfyui-truss => image/comfyui}/model/helpers.py (100%) rename {comfyui-truss => image/comfyui}/model/model.py (100%) create mode 100644 image/control-net-qrcode/README.md rename {control-net-qrcode => image/control-net-qrcode}/config.yaml (82%) rename {control-net-qrcode => image/control-net-qrcode}/controlnet_qr_code_results.gif (100%) rename {flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3 => image/control-net-qrcode/model}/__init__.py (100%) rename {control-net-qrcode => image/control-net-qrcode}/model/model.py (100%) rename {control-net-qrcode => image/control-net-qrcode}/twitter_mask.jpeg (100%) rename {control-net-qrcode => image/control-net-qrcode}/twitter_output.jpg (100%) create mode 100644 image/deepfloyd-xl/README.md rename {deepfloyd-xl => image/deepfloyd-xl}/config.yaml (62%) rename {flux/dev => image/deepfloyd-xl}/model/__init__.py (100%) rename {deepfloyd-xl => image/deepfloyd-xl}/model/model.py (100%) create mode 100644 image/dis-segmentation/README.md rename {dis-segmentation => image/dis-segmentation}/config.yaml (80%) rename {flux/schnell => image/dis-segmentation}/model/__init__.py (100%) rename {dis-segmentation => image/dis-segmentation}/model/clone_repo_helper.py (100%) rename {dis-segmentation => image/dis-segmentation}/model/helpers.py (100%) rename {dis-segmentation => image/dis-segmentation}/model/model.py (100%) create mode 100644 image/flux-dev-trt-b200/README.md rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/config.yaml (76%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/load_test.py (100%) rename {gemma/gemma-2-27b-it-vllm => image/flux-dev-trt-b200}/model/__init__.py (100%) rename {gemma/gemma-2-9b-it-vllm/model => image/flux-dev-trt-b200/model/demo_diffusion}/__init__.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/dd_argparse.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/dynamic_import.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/engine.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/image/__init__.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/image/load.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/image/resize.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/image/video.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/__init__.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/base_model.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/clip.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/diffusion_transformer.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/gan.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/load.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/lora.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/optimizer.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/scheduler.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/t5.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/tokenizer.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/unet.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/model/vae.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/path/__init__.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/path/dd_path.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/path/resolve_path.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/__init__.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/calibrate.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/diffusion_pipeline.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/flux_pipeline.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/model_memory_manager.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/stable_cascade_pipeline.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/stable_diffusion_35_pipeline.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/stable_diffusion_3_pipeline.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/stable_diffusion_pipeline.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/stable_video_diffusion_pipeline.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/pipeline/type.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/utils_modelopt.py (100%) rename {gfp-gan/model => image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3}/__init__.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/utils_sd3/mmdit.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/utils_sd3/other_impls.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/demo_diffusion/utils_sd3/sd3_impls.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/model/model.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/requirements.txt (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/show.py (100%) rename {flux.1-dev-trt-b200 => image/flux-dev-trt-b200}/show_batch.py (100%) create mode 100644 image/flux/README.md create mode 100644 image/flux/dev/README.md rename {flux => image/flux}/dev/config.yaml (76%) rename {image-segmentation => image/flux/dev}/model/__init__.py (100%) rename {flux => image/flux}/dev/model/model.py (100%) create mode 100644 image/flux/schnell/README.md rename {flux => image/flux}/schnell/config.yaml (73%) rename {ip-adapter => image/flux/schnell}/model/__init__.py (100%) rename {flux => image/flux}/schnell/model/model.py (100%) create mode 100644 image/fotographer/zenctrl/README.md rename {fotographer => image/fotographer}/zenctrl/call.py (100%) rename {fotographer => image/fotographer}/zenctrl/config.yaml (68%) rename {fotographer => image/fotographer}/zenctrl/images/banner_1.png (100%) rename {fotographer => image/fotographer}/zenctrl/images/camera.png (100%) rename {fotographer => image/fotographer}/zenctrl/images/speaker-input.png (100%) rename {fotographer => image/fotographer}/zenctrl/images/speaker-output.png (100%) rename {fotographer => image/fotographer}/zenctrl/requirements.txt (100%) rename {whisper/whisper-truss => image/gfp-gan}/LICENSE (100%) create mode 100644 image/gfp-gan/README.md rename {gfp-gan => image/gfp-gan}/config.yaml (90%) rename autodesk-wala/packages/src/diffusion_modules/modules/__init__ copy.py => image/gfp-gan/data/.gitkeep (100%) rename {gfp-gan => image/gfp-gan}/input.json (100%) rename {jsonformatter => image/gfp-gan}/model/__init__.py (100%) rename {gfp-gan => image/gfp-gan}/model/model.py (100%) create mode 100644 image/image-segmentation/README.md create mode 100644 image/image-segmentation/config.yaml rename {kokoro => image/image-segmentation}/model/__init__.py (100%) rename {image-segmentation => image/image-segmentation}/model/model.py (100%) create mode 100644 image/ip-adapter/README.md rename {ip-adapter => image/ip-adapter}/config.yaml (64%) rename {layoutlm-document-qa => image/ip-adapter}/model/__init__.py (100%) rename {ip-adapter => image/ip-adapter}/model/model.py (100%) create mode 100644 image/magic-animate/README.md rename {magic-animate => image/magic-animate}/config.yaml (87%) rename {llama/llama-2-13b-chat => image/magic-animate}/model/__init__.py (100%) rename {magic-animate => image/magic-animate}/model/configs/inference/inference.yaml (100%) rename {magic-animate => image/magic-animate}/model/configs/prompts/animation.yaml (100%) rename {magic-animate => image/magic-animate}/model/demo/animate.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/appearance_encoder.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/attention.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/controlnet.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/embeddings.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/motion_module.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/mutual_self_attention.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/orig_attention.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/resnet.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/stable_diffusion_controlnet_reference.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/unet.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/unet_3d_blocks.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/models/unet_controlnet.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/pipelines/animation.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/pipelines/context.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/pipelines/pipeline_animation.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/utils/dist_tools.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/utils/util.py (100%) rename {magic-animate => image/magic-animate}/model/magicanimate/utils/videoreader.py (100%) rename {magic-animate => image/magic-animate}/model/model.py (100%) create mode 100644 image/playground-v2-aesthetic/README.md rename {playground-v2-aesthetic => image/playground-v2-aesthetic}/config.yaml (100%) rename {llama/llama-2-13b => image/playground-v2-aesthetic}/model/__init__.py (100%) rename {playground-v2-aesthetic => image/playground-v2-aesthetic}/model/model.py (100%) rename {playground-v2-aesthetic => image/playground-v2-aesthetic}/show.py (100%) create mode 100644 image/sana/sana-1600m/README.md rename {sana/sana_1600M => image/sana/sana-1600m}/config.yaml (86%) rename {llama/llama-2-70b-chat => image/sana/sana-1600m}/model/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/model/model.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/CITATION.bib (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/CIs/add_license_all.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/Dockerfile (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/LICENSE (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/README.md (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/app/app_sana.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/app/app_sana_multithread.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/app/safety_check.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/app/sana_pipeline.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/Sana.jpg (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/docs/metrics_toolkit.md (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/00000000.png (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/00000000.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/00000000_InternVL2-26B.json (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/00000000_prompt_clip_score.json (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/example_data/meta_data.json (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/examples.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/logo.png (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/model-incremental.jpg (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/model_paths.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/samples.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/asset/samples_mini.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_base.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/builder.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/datasets/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/datasets/sana_data.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/datasets/utils.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/transforms.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/wids/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/wids/wids.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/wids/wids_dl.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/wids/wids_lru.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/wids/wids_mmtar.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/wids/wids_specs.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/data/wids/wids_tar.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/dpm_solver.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/flow_euler_sampler.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/iddpm.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/lcm_scheduler.py (100%) rename {llama/llama-2-70b => image/sana/sana-1600m/packages/Sana/diffusion}/model/__init__.py (100%) mode change 100644 => 100755 rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/act.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/builder.py (100%) rename {llama/llama-2-7b-chat/model => image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit}/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py (100%) rename {llama/llama-2-7b/model => image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps}/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py (100%) rename {llama/llama-3-70b-instruct/model => image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models}/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/diffusion_utils.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/dpm_solver.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/edm_sample.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/gaussian_diffusion.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/basic_modules.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/fastlinear/readme.md (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/sana.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/sana_U_shape.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/sana_blocks.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/sana_multi_scale.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/nets/sana_others.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/norms.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/respace.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/sa_solver.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/timestep_sampler.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/model/utils.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/sa_sampler.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/sa_solver_diffusers.py (100%) rename {llama/llama-3-8b-instruct/model => image/sana/sana-1600m/packages/Sana/diffusion/utils}/__init__.py (100%) mode change 100644 => 100755 rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/checkpoint.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/config.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/data_sampler.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/dist_utils.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/import_utils.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/logger.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/lr_scheduler.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/misc.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/diffusion/utils/optimizer.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/environment_setup.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/pyproject.toml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/sana/cli/run.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/sana/cli/upload2hf.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/sana/tools/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/sana/tools/download.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/sana/tools/hf_utils.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/bash_run_inference_metric.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/bash_run_inference_metric_dpg.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/bash_run_inference_metric_geneval.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/infer_metric_run_inference_metric.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/infer_run_inference.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/infer_run_inference_geneval.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/inference.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/inference_dpg.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/inference_geneval.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/inference_geneval_diffusers.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/inference_image_reward.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/interface.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/scripts/style.css (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tests/bash/entry.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tests/bash/test_inference.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tests/bash/test_training_1epoch.sh (100%) rename {llama/llama-3_1-405b-instruct/model => image/sana/sana-1600m/packages/Sana/tools}/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/convert_py_to_yaml.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/convert_sana_pag_to_diffusers.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/convert_sana_to_diffusers.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/download.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/.gitignore (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/LICENSE (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/README.md (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/clip_score.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/setup.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/compute_clipscore.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/compute_dpg.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/compute_fid_embedding.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/compute_geneval.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/compute_imagereward.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/dpg_bench/requirements.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/LICENSE (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/README.md (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/environment.yml (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/geneval/prompts/object_names.txt (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/image_reward/compute_image_reward.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/.gitignore (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/LICENSE (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/README.md (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/noxfile.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/setup.cfg (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/setup.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/tools/metrics/utils.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/train_scripts/train.py (100%) rename {sana/sana_1600M => image/sana/sana-1600m}/packages/Sana/train_scripts/train.sh (100%) create mode 100644 image/sana/sana-600m/README.md rename {sana/sana_600M => image/sana/sana-600m}/config.yaml (87%) rename {llama/llama-3_1-8b-instruct-sglang => image/sana/sana-600m}/model/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/model/model.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/CITATION.bib (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/CIs/add_license_all.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/Dockerfile (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/LICENSE (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/README.md (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/app/app_sana.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/app/app_sana_multithread.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/app/safety_check.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/app/sana_pipeline.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/Sana.jpg (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/docs/metrics_toolkit.md (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/00000000.png (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/00000000.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/00000000_InternVL2-26B.json (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/00000000_prompt_clip_score.json (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/example_data/meta_data.json (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/examples.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/logo.png (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/model-incremental.jpg (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/model_paths.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/samples.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/asset/samples_mini.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_base.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/builder.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/datasets/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/datasets/sana_data.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/datasets/utils.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/transforms.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/wids/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/wids/wids.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/wids/wids_dl.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/wids/wids_lru.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/wids/wids_mmtar.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/wids/wids_specs.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/data/wids/wids_tar.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/dpm_solver.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/flow_euler_sampler.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/iddpm.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/lcm_scheduler.py (100%) rename {llama/llama-3_1-8b-instruct => image/sana/sana-600m/packages/Sana/diffusion}/model/__init__.py (100%) mode change 100644 => 100755 rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/act.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/builder.py (100%) rename {llama/llama-3_1_70b-instruct/model => image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit}/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py (100%) rename {llama/llama-7b-exllama-streaming/model => image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps}/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py (100%) rename {llama/llama-7b-exllama/model => image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models}/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/diffusion_utils.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/dpm_solver.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/edm_sample.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/gaussian_diffusion.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/basic_modules.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/fastlinear/readme.md (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/sana.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/sana_U_shape.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/sana_blocks.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/sana_multi_scale.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/nets/sana_others.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/norms.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/respace.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/sa_solver.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/timestep_sampler.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/model/utils.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/sa_sampler.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/sa_solver_diffusers.py (100%) rename {llama/llama-7b-vllm/model => image/sana/sana-600m/packages/Sana/diffusion/utils}/__init__.py (100%) mode change 100644 => 100755 rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/checkpoint.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/config.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/data_sampler.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/dist_utils.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/import_utils.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/logger.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/lr_scheduler.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/misc.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/diffusion/utils/optimizer.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/environment_setup.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/pyproject.toml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/sana/cli/run.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/sana/cli/upload2hf.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/sana/tools/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/sana/tools/download.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/sana/tools/hf_utils.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/bash_run_inference_metric.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/bash_run_inference_metric_dpg.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/bash_run_inference_metric_geneval.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/infer_metric_run_inference_metric.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/infer_run_inference.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/infer_run_inference_geneval.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/inference.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/inference_dpg.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/inference_geneval.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/inference_geneval_diffusers.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/inference_image_reward.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/interface.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/scripts/style.css (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tests/bash/entry.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tests/bash/test_inference.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tests/bash/test_training_1epoch.sh (100%) rename {llama/llama-7b/model => image/sana/sana-600m/packages/Sana/tools}/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/convert_py_to_yaml.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/convert_sana_pag_to_diffusers.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/convert_sana_to_diffusers.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/download.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/.gitignore (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/LICENSE (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/README.md (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/clip_score.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/setup.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/compute_clipscore.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/compute_dpg.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/compute_fid_embedding.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/compute_geneval.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/compute_imagereward.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/dpg_bench/requirements.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/LICENSE (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/README.md (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/environment.yml (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/geneval/prompts/object_names.txt (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/image_reward/compute_image_reward.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/.gitignore (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/LICENSE (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/README.md (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/noxfile.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/setup.cfg (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/setup.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/tools/metrics/utils.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/train_scripts/train.py (100%) rename {sana/sana_600M => image/sana/sana-600m}/packages/Sana/train_scripts/train.sh (100%) create mode 100644 image/segment-anything/README.md rename {segment-anything => image/segment-anything}/config.yaml (88%) rename {llava/llava-1.6-sgl => image/segment-anything}/model/__init__.py (100%) rename {segment-anything => image/segment-anything}/model/model.py (100%) create mode 100644 image/stable-diffusion/dreamshaper-lcm/README.md create mode 100644 image/stable-diffusion/dreamshaper-lcm/config.yaml rename {llava/llava-v1.5-7b => image/stable-diffusion/dreamshaper-lcm}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/dreamshaper-lcm/model/model.py (100%) create mode 100644 image/stable-diffusion/playground-v2-trt/README.md rename {stable-diffusion => image/stable-diffusion}/playground-v2-trt/config.yaml (85%) rename {llava/llava-v1.5-7b/packages/llava/serve => image/stable-diffusion/playground-v2-trt/model}/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/playground-v2-trt/model/model.py (100%) rename {stable-diffusion => image/stable-diffusion}/playground-v2-trt/packages/diffusion/trtclip.py (100%) rename {stable-diffusion => image/stable-diffusion}/playground-v2-trt/packages/diffusion/trtunet.py (100%) rename {stable-diffusion => image/stable-diffusion}/playground-v2-trt/show.py (100%) create mode 100644 image/stable-diffusion/sd-textual-inversion/README.md rename {stable-diffusion => image/stable-diffusion}/sd-textual-inversion/config.yaml (51%) rename {stable-diffusion => image/stable-diffusion}/sd-textual-inversion/data/LulaCipher.bin (100%) rename {llava/llava-v1.6-34b => image/stable-diffusion/sd-textual-inversion}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sd-textual-inversion/model/model.py (100%) create mode 100644 image/stable-diffusion/sd-turbo/README.md rename {stable-diffusion => image/stable-diffusion}/sd-turbo/config.yaml (91%) rename {magic-animate => image/stable-diffusion/sd-turbo}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sd-turbo/model/model.py (100%) create mode 100644 image/stable-diffusion/sdxl-controlnet-canny/README.md rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet-canny/baseten-logo.gif (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet-canny/config.yaml (82%) rename {metavoice-1b => image/stable-diffusion/sdxl-controlnet-canny}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet-canny/model/model.py (100%) create mode 100644 image/stable-diffusion/sdxl-controlnet-depth/README.md rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet-depth/config.yaml (82%) rename {midnight => image/stable-diffusion/sdxl-controlnet-depth}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet-depth/model/model.py (100%) create mode 100644 image/stable-diffusion/sdxl-controlnet/README.md rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet/baseten-logo.gif (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet/config.yaml (69%) rename {mistral/mistral-7b-chat => image/stable-diffusion/sdxl-controlnet}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-controlnet/model/model.py (100%) create mode 100644 image/stable-diffusion/sdxl-lightning/README.md rename {stable-diffusion => image/stable-diffusion}/sdxl-lightning/config.yaml (87%) rename {mistral/mistral-7b-instruct-vllm => image/stable-diffusion/sdxl-lightning}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-lightning/model/model.py (100%) create mode 100644 image/stable-diffusion/sdxl-lora-swapping/README.md rename {stable-diffusion => image/stable-diffusion}/sdxl-lora-swapping/config.yaml (81%) rename {mistral/mistral-7b-instruct => image/stable-diffusion/sdxl-lora-swapping}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-lora-swapping/model/model.py (100%) create mode 100644 image/stable-diffusion/sdxl-lora/README.md create mode 100644 image/stable-diffusion/sdxl-lora/config.yaml rename {mistral/mistral-7b => image/stable-diffusion/sdxl-lora}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-lora/model/model.py (100%) create mode 100644 image/stable-diffusion/sdxl-turbo/README.md rename {stable-diffusion => image/stable-diffusion}/sdxl-turbo/config.yaml (92%) rename {mistral/mixtral-8x22b-trt-int8-weights-only => image/stable-diffusion/sdxl-turbo}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/sdxl-turbo/model/model.py (100%) create mode 100644 image/stable-diffusion/stable-diffusion-3-medium/README.md rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-3-medium/config.yaml (59%) rename {mistral/mixtral-8x22b => image/stable-diffusion/stable-diffusion-3-medium}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-3-medium/model/model.py (100%) create mode 100644 image/stable-diffusion/stable-diffusion-inpainting-trt/README.md create mode 100644 image/stable-diffusion/stable-diffusion-inpainting-trt/config.yaml rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => image/stable-diffusion/stable-diffusion-inpainting-trt}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-inpainting-trt/model/model.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-inpainting-trt/packages/helpers/inpaint_pipeline.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-inpainting-trt/packages/helpers/models.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-inpainting-trt/packages/helpers/stable_diffusion_pipeline.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-inpainting-trt/packages/helpers/utilities.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-inpainting-trt/requirements.txt (100%) create mode 100644 image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt-h100/config.yaml (86%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt-h100/model/model.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtclip.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtunet.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt-h100/show.py (100%) create mode 100644 image/stable-diffusion/stable-diffusion-xl-1.0-trt/README.md rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt/config.yaml (86%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant => image/stable-diffusion/stable-diffusion-xl-1.0-trt}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt/model/model.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt/packages/diffusion/trtclip.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt/packages/diffusion/trtunet.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0-trt/show.py (100%) create mode 100644 image/stable-diffusion/stable-diffusion-xl-1.0/README.md rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0/config.yaml (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm => image/stable-diffusion/stable-diffusion-xl-1.0}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0/model/model.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion-xl-1.0/show.py (100%) create mode 100644 image/stable-diffusion/stable-diffusion/README.md rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/config.yaml (79%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/model_index.json (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/scheduler/scheduler_config.json (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/text_encoder/config.json (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/tokenizer/merges.txt (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/tokenizer/special_tokens_map.json (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/tokenizer/tokenizer_config.json (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/tokenizer/vocab.json (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/unet/config.json (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/data/vae/config.json (100%) rename {mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2 => image/stable-diffusion/stable-diffusion}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/model/model.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-diffusion/show.py (100%) create mode 100644 image/stable-diffusion/stable-video-diffusion/README.md rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/config.yaml (91%) rename {mistral/mixtral-8x7b-instruct-vllm => image/stable-diffusion/stable-video-diffusion}/model/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/helper.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/model.py (100%) rename {mistral/pixtral-12b/model => image/stable-diffusion/stable-video-diffusion/model/scripts}/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/sampling/configs/svd.yaml (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/sampling/configs/svd_image_decoder.yaml (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/sampling/configs/svd_xt.yaml (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/sampling/configs/svd_xt_image_decoder.yaml (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/sampling/simple_video_sample.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/tests/attention.py (100%) rename {multiprocessing/model => image/stable-diffusion/stable-video-diffusion/model/scripts/util}/__init__.py (100%) rename {musicgen-large/model => image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection}/__init__.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/util/detection/nsfw_and_watermark_dectection.py (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/util/detection/p_head_v1.npz (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/model/scripts/util/detection/w_head_v1.npz (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/sample_images/cheetah.jpeg (100%) rename {stable-diffusion => image/stable-diffusion}/stable-video-diffusion/sample_images/racecar.jpeg (100%) create mode 100644 infrastructure/README.md create mode 100644 infrastructure/autodesk-wala/README.md rename {autodesk-wala => infrastructure/autodesk-wala}/config.yaml (54%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/README.md (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_4/10.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_4/26.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_4/3.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_4/6.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_6/10.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_6/26.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_6/3.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_6/49.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_6/50.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/depth_maps_6/6.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/multi_view/003.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/multi_view/006.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/multi_view/010.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/multi_view/026.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/pointcloud/ring.h5df (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/single_depth_map/49.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/single_view/table.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/sketch/tree.png (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/voxel/horse_16.json (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/examples/voxel/horse_32.json (100%) rename {musicgen-melody => infrastructure/autodesk-wala}/model/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/model/model.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/output.obj (100%) rename {ngram-speculator/truss/model => infrastructure/autodesk-wala/packages/src}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/bpe_simple_vocab_16e6.txt.gz (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/clip_mod.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/dataset_utils.py (100%) rename {nous-capybara/nous-capybara-34b-openai/model => infrastructure/autodesk-wala/packages/src/diffusion_modules}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/dwt.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/dwt_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/fp16_util.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/gaussian_diffusion.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/latent_points.py (100%) rename nous-capybara/nous-capybara-34b/model/__init__.py => infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/__init__ copy.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/ball_query.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/frustum.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/backend.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/ball_query.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/devoxelization.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/grouping.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/interpolatation.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/loss.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/sampling.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cu (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cuh (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.hpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/bindings.cpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/cuda_utils.cuh (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cu (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cuh (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.hpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cu (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cuh (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.hpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cu (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cuh (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.hpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cu (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cuh (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.hpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/utils.hpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cu (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cuh (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.hpp (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/functional/voxelization.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/loss.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/pointnet.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/pvconv.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/se.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/shared_mlp.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/modules/voxelization.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/network_ae.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/nn.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/point_voxels.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/resample.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/diffusion_modules/sparse_network.py (100%) rename {nsql/model => infrastructure/autodesk-wala/packages/src/experiments}/__init__.py (100%) rename {nvidia/parakeet-tdt-0_6b-v2/model => infrastructure/autodesk-wala/packages/src/experiments/utils}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/experiments/utils/wavelet_utils.py (100%) rename {phi/phi-3-mini-128k-instruct/model => infrastructure/autodesk-wala/packages/src/latent_model}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/abstract_volume_nn.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/continous_diffusion_interface.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/dwt.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/dwt_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/fp16_util.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/gaussian_diffusion.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/gaussian_diffusion_wavelet.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/nn.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/resample.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/sparse_network.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/diffusion_modules/utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/dit_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/gaussian_diffusion.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/latent_dit_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/latent_uvit_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/points_network.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/quantize.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/voxels_network.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/wavelet_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_model/wavelet_vq_model.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/latent_module.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/model_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/callbacks.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/camera_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/coco_prompts.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/configs/sd-v1.yaml (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/configs/sd-v2-base.yaml (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/constants.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/helper.py (100%) rename {phi/phi-3-mini-4k-instruct/model => infrastructure/autodesk-wala/packages/src/mvdream/ldm}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/interface.py (100%) rename {phi/phi-3.5-mini/model => infrastructure/autodesk-wala/packages/src/mvdream/ldm/models}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/models/autoencoder.py (100%) rename {piper-tts/model => infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/models/diffusion/ddim.py (100%) rename {playground-v2-aesthetic/model => infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/modules/attention.py (100%) rename {qwen/qwen-7b-chat/model => infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/modules/diffusionmodules/model.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/modules/diffusionmodules/openaimodel.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/modules/diffusionmodules/util.py (100%) rename {qwen/qwen-vl/model => infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/distributions}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/modules/distributions/distributions.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/modules/ema.py (100%) rename {sana/sana_1600M/model => infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/encoders}/__init__.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/modules/encoders/modules.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/ldm/util.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/model_zoo.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/trainer_old.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream/trainermodule.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream_module.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/mvdream_utils.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/networks/callbacks.py (100%) rename {sana/sana_1600M/packages/Sana/diffusion/model => infrastructure/autodesk-wala/packages/src/utils}/__init__.py (100%) mode change 100755 => 100644 rename {autodesk-wala => infrastructure/autodesk-wala}/packages/src/utils/visualization.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/requirements.txt (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/test.py (100%) rename {autodesk-wala => infrastructure/autodesk-wala}/vendor_wala.sh (100%) create mode 100644 infrastructure/binocular/README.md rename {binocular => infrastructure/binocular}/config.yaml (63%) rename {sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit => infrastructure/binocular/model}/__init__.py (100%) rename {binocular => infrastructure/binocular}/model/model.py (100%) rename {binocular => infrastructure/binocular}/packages/config.py (100%) rename {chains-examples => infrastructure/chains-examples}/docs/audio-transcription/README.md (100%) rename {chains-examples => infrastructure/chains-examples}/docs/audio-transcription/data_types.py (100%) rename {chains-examples => infrastructure/chains-examples}/docs/audio-transcription/helpers.py (100%) rename {chains-examples => infrastructure/chains-examples}/docs/audio-transcription/transcribe.py (100%) rename {chains-examples => infrastructure/chains-examples}/docs/audio-transcription/whisper_chainlet.py (100%) rename {chains-examples => infrastructure/chains-examples}/docs/poems/poems.py (100%) create mode 100644 infrastructure/custom-engine-builder-control/README.md rename {custom-engine-builder-control => infrastructure/custom-engine-builder-control}/config.yaml (95%) rename {custom-engine-builder-control => infrastructure/custom-engine-builder-control}/model/model.py (100%) create mode 100644 infrastructure/custom-server/README.md create mode 100644 infrastructure/custom-server/deepseek-v2-5-instruct-sglang/README.md rename {custom-server => infrastructure/custom-server}/deepseek-v2-5-instruct-sglang/config.yaml (76%) create mode 100644 infrastructure/custom-server/infinity-embedding-server/README.md rename {custom-server => infrastructure/custom-server}/infinity-embedding-server/config.yaml (83%) create mode 100644 infrastructure/custom-server/llama3-70b-instruct-lmdeploy/README.md rename {custom-server => infrastructure/custom-server}/llama3-70b-instruct-lmdeploy/config.yaml (75%) create mode 100644 infrastructure/custom-server/llama3-70b-instruct-sglang/README.md rename {custom-server => infrastructure/custom-server}/llama3-70b-instruct-sglang/config.yaml (75%) create mode 100644 infrastructure/custom-server/llama3-8b-instruct-lmdeploy/README.md rename {custom-server => infrastructure/custom-server}/llama3-8b-instruct-lmdeploy/config.yaml (74%) create mode 100644 infrastructure/custom-server/llama3-8b-instruct-sglang/README.md rename {custom-server => infrastructure/custom-server}/llama3-8b-instruct-sglang/config.yaml (75%) rename {custom-server => infrastructure/custom-server}/llama3_eval.py (100%) create mode 100644 infrastructure/custom-server/pixtral-12b/README.md rename {custom-server => infrastructure/custom-server}/pixtral-12b/config.yaml (96%) rename {custom-server => infrastructure/custom-server}/pixtral-12b/data/pixtral12b.jinja (100%) create mode 100644 infrastructure/custom-server/ultravox-0.4/README.md rename {custom-server => infrastructure/custom-server}/ultravox-0.4/config.yaml (100%) create mode 100644 infrastructure/custom-server/ultravox-0.5-8b/README.md rename {custom-server => infrastructure/custom-server}/ultravox-0.5-8b/config.yaml (100%) create mode 100644 infrastructure/custom-server/ultravox-0.6-70b/README.md rename {custom-server => infrastructure/custom-server}/ultravox-0.6-70b/config.yaml (100%) create mode 100644 infrastructure/custom-server/voxtral-mini-3b-2507/README.md rename {custom-server => infrastructure/custom-server}/voxtral-mini-3b-2507/config.yaml (98%) create mode 100644 infrastructure/custom-server/voxtral-small-24b-2507/README.md rename {custom-server => infrastructure/custom-server}/voxtral-small-24b-2507/config.yaml (98%) rename {grpc => infrastructure/grpc}/Dockerfile (100%) create mode 100644 infrastructure/grpc/README.md rename {grpc => infrastructure/grpc}/client.py (100%) rename {grpc => infrastructure/grpc}/config.yaml (75%) rename {grpc => infrastructure/grpc}/example.proto (100%) rename {grpc => infrastructure/grpc}/example_pb2.py (100%) rename {grpc => infrastructure/grpc}/example_pb2_grpc.py (100%) rename {grpc => infrastructure/grpc}/model.py (100%) rename {grpc => infrastructure/grpc}/requirements.txt (100%) create mode 100644 infrastructure/jsonformatter/README.md rename {jsonformatter => infrastructure/jsonformatter}/config.yaml (50%) rename {jsonformatter => infrastructure/jsonformatter}/data/schema.json (100%) rename {sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps => infrastructure/jsonformatter/model}/__init__.py (100%) rename {jsonformatter => infrastructure/jsonformatter}/model/model.py (100%) create mode 100644 infrastructure/layoutlm-document-qa/README.md rename {layoutlm-document-qa => infrastructure/layoutlm-document-qa}/config.yaml (94%) rename {sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models => infrastructure/layoutlm-document-qa/model}/__init__.py (100%) rename {layoutlm-document-qa => infrastructure/layoutlm-document-qa}/model/model.py (100%) create mode 100644 infrastructure/llama-cpp-server/README.md rename {llama-cpp-server => infrastructure/llama-cpp-server}/config.yaml (97%) rename {llama-cpp-server => infrastructure/llama-cpp-server}/cuda.Dockerfile (100%) rename {llama-cpp-server => infrastructure/llama-cpp-server}/llama_server_help (100%) rename {metrics => infrastructure/metrics}/datadog/Dockerfile (100%) create mode 100644 infrastructure/metrics/datadog/README.md rename {metrics => infrastructure/metrics}/datadog/config.yaml (95%) rename {metrics => infrastructure/metrics}/datadog/datadog-vllm-guide.md (100%) rename {metrics => infrastructure/metrics}/datadog/vllm_conf.yaml (100%) create mode 100644 infrastructure/model-cache/README.md rename {model_cache => infrastructure/model-cache}/config.yaml (84%) rename {model_cache => infrastructure/model-cache}/model/model.py (100%) create mode 100644 infrastructure/multiprocessing/README.md rename {multiprocessing => infrastructure/multiprocessing}/config.yaml (65%) rename {sana/sana_1600M/packages/Sana/diffusion/utils => infrastructure/multiprocessing/model}/__init__.py (100%) mode change 100755 => 100644 rename {multiprocessing => infrastructure/multiprocessing}/model/model.py (100%) rename {multiprocessing => infrastructure/multiprocessing}/model/test.py (100%) create mode 100644 infrastructure/ngram-speculator/truss/README.md rename {ngram-speculator => infrastructure/ngram-speculator}/truss/config.yaml (50%) rename {sana/sana_1600M/packages/Sana/tools => infrastructure/ngram-speculator/truss/model}/__init__.py (100%) rename {ngram-speculator => infrastructure/ngram-speculator}/truss/model/model.py (100%) create mode 100644 infrastructure/ngram-speculator/trussless/README.md rename {ngram-speculator => infrastructure/ngram-speculator}/trussless/config.yaml (92%) create mode 100644 infrastructure/paddlepaddle/paddleocr-vl/README.md rename {paddlepaddle/PaddleOCR_VL => infrastructure/paddlepaddle/paddleocr-vl}/config.yaml (95%) delete mode 100644 ip-adapter/README.md delete mode 100644 kokoro/README.md delete mode 100644 layoutlm-document-qa/README.md delete mode 100644 llama-cpp-server/README.md delete mode 100644 llama/engine-llama-3-1-70b-instruct/README.md delete mode 100644 llama/engine-llama-3-1-8b-instruct/README.md delete mode 100644 llama/engine-llama-3-3-70b-instruct/README.md delete mode 100644 llama/engine-llama-3.1-405b-instruct/README.md delete mode 100644 llama/llama-3-70b-instruct/README.md delete mode 100644 llama/llama-3-8b-instruct/README.md delete mode 100644 llama/llama-3_1-405b-instruct/README.md delete mode 100644 llama/llama-3_2-11b-vision-instruct/README.md delete mode 100644 llava/llava-1.6-sgl/README.md delete mode 100644 llava/llava-v1.5-7b/README.md delete mode 100644 llava/llava-v1.6-34b/README.md delete mode 100644 llava/llava-v1.6-34b/config.yaml create mode 100644 llm/README.md create mode 100644 llm/cogito/README.md create mode 100644 llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md rename {cogito => llm/cogito}/cogito-v2-preview-deepseek-671B-MoE-vllm/config.yaml (94%) create mode 100644 llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md rename {cogito => llm/cogito}/cogito-v2-preview-llama-109B-MoE-vllm/config.yaml (94%) create mode 100644 llm/cogito/cogito-v2-preview-llama-405B-vllm/README.md rename {cogito => llm/cogito}/cogito-v2-preview-llama-405B-vllm/config.yaml (95%) create mode 100644 llm/cogito/cogito-v2-preview-llama-70B-vllm/README.md rename {cogito => llm/cogito}/cogito-v2-preview-llama-70B-vllm/config.yaml (94%) create mode 100644 llm/cogvlm/README.md rename {cogvlm => llm/cogvlm}/config.yaml (59%) rename {sana/sana_600M => llm/cogvlm}/model/__init__.py (100%) rename {cogvlm => llm/cogvlm}/model/model.py (100%) rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/Bad-Handwriting.png (100%) create mode 100644 llm/deepseek/deepseek-ocr/README.md rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/config.yaml (91%) rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/model/__init__.py (100%) rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/model/model.py (100%) rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/test_document_ocr.py (100%) rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/visualization_1.png (100%) rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/visualization_2.png (100%) rename {deepseek-ocr => llm/deepseek/deepseek-ocr}/visualizer.py (100%) create mode 100644 llm/deepseek/deepseek-vl2/README.md rename {deepseek => llm/deepseek}/deepseek-vl2/config.yaml (91%) create mode 100644 llm/deepseek/engine-deepseek-r1-distill-llama-70b/README.md rename {deepseek => llm/deepseek}/engine-deepseek-r1-distill-llama-70b/config.yaml (94%) create mode 100644 llm/deepseek/engine-deepseek-r1-distill-llama-8b/README.md rename {deepseek => llm/deepseek}/engine-deepseek-r1-distill-llama-8b/config.yaml (94%) create mode 100644 llm/deepseek/engine-deepseek-r1-distill-qwen-14b/README.md rename {deepseek => llm/deepseek}/engine-deepseek-r1-distill-qwen-14b/config.yaml (93%) create mode 100644 llm/deepseek/engine-deepseek-r1-distill-qwen-32b/README.md rename {deepseek => llm/deepseek}/engine-deepseek-r1-distill-qwen-32b/config.yaml (93%) create mode 100644 llm/deepseek/engine-deepseek-r1-distill-qwen-7b/README.md rename {deepseek => llm/deepseek}/engine-deepseek-r1-distill-qwen-7b/config.yaml (93%) create mode 100644 llm/falcon/_archive/falcon3-10B-trt-llm-spec-dec/README.md rename {falcon => llm/falcon/_archive}/falcon3-10B-trt-llm-spec-dec/config.yaml (100%) create mode 100644 llm/falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md rename {falcon => llm/falcon}/falcon3-3B-trt-llm-engine-high-throughput/config.yaml (91%) create mode 100644 llm/gemma/gemma-2-27b-it-vllm/README.md rename {gemma => llm/gemma}/gemma-2-27b-it-vllm/config.yaml (89%) rename {sana/sana_600M/packages/Sana/diffusion => llm/gemma/gemma-2-27b-it-vllm}/model/__init__.py (100%) mode change 100755 => 100644 rename {gemma => llm/gemma}/gemma-2-27b-it-vllm/model/model.py (100%) create mode 100644 llm/gemma/gemma-2-9b-it-vllm/README.md rename {gemma => llm/gemma}/gemma-2-9b-it-vllm/config.yaml (89%) rename {sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit => llm/gemma/gemma-2-9b-it-vllm/model}/__init__.py (100%) rename {gemma => llm/gemma}/gemma-2-9b-it-vllm/model/model.py (100%) create mode 100644 llm/gemma/gemma-3-27b-it/README.md rename {gemma => llm/gemma}/gemma-3-27b-it/config.yaml (93%) rename {llama => llm/llama/_archive}/llama-2-13b-chat/README.md (100%) rename {llama => llm/llama/_archive}/llama-2-13b-chat/config.yaml (100%) rename {sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps => llm/llama/_archive/llama-2-13b-chat/model}/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-2-13b-chat/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-2-13b/README.md (100%) rename {llama => llm/llama/_archive}/llama-2-13b/config.yaml (100%) rename {sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models => llm/llama/_archive/llama-2-13b/model}/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-2-13b/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-2-70b-chat/README.md (100%) rename {llama => llm/llama/_archive}/llama-2-70b-chat/config.yaml (100%) rename {sana/sana_600M/packages/Sana/diffusion/utils => llm/llama/_archive/llama-2-70b-chat/model}/__init__.py (100%) mode change 100755 => 100644 rename {llama => llm/llama/_archive}/llama-2-70b-chat/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-2-70b/README.md (100%) rename {llama => llm/llama/_archive}/llama-2-70b/config.yaml (100%) rename {sana/sana_600M/packages/Sana/tools => llm/llama/_archive/llama-2-70b/model}/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-2-70b/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-2-7b-chat/README.md (100%) rename {llama => llm/llama/_archive}/llama-2-7b-chat/config.yaml (100%) rename {segment-anything => llm/llama/_archive/llama-2-7b-chat}/model/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-2-7b-chat/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-2-7b/README.md (100%) rename {llama => llm/llama/_archive}/llama-2-7b/config.yaml (100%) rename {sesame-csm-1b => llm/llama/_archive/llama-2-7b}/model/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-2-7b/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-7b-exllama-streaming/README.md (100%) rename {llama => llm/llama/_archive}/llama-7b-exllama-streaming/config.yaml (100%) rename {stable-diffusion/dreamshaper-lcm => llm/llama/_archive/llama-7b-exllama-streaming}/model/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-7b-exllama-streaming/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-7b-exllama/README.md (100%) rename {llama => llm/llama/_archive}/llama-7b-exllama/config.yaml (100%) rename {stable-diffusion/playground-v2-trt => llm/llama/_archive/llama-7b-exllama}/model/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-7b-exllama/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-7b-vllm/config.yaml (100%) rename {stable-diffusion/sd-textual-inversion => llm/llama/_archive/llama-7b-vllm}/model/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-7b-vllm/model/model.py (100%) rename {llama => llm/llama/_archive}/llama-7b/README.md (100%) rename {llama => llm/llama/_archive}/llama-7b/config.yaml (100%) rename {llama => llm/llama/_archive}/llama-7b/data/config.json (100%) rename {llama => llm/llama/_archive}/llama-7b/data/generation_config.json (100%) rename {llama => llm/llama/_archive}/llama-7b/data/pytorch_model.bin.index.json (100%) rename {stable-diffusion/sd-turbo => llm/llama/_archive/llama-7b}/model/__init__.py (100%) rename {llama => llm/llama/_archive}/llama-7b/model/model.py (100%) create mode 100644 llm/llama/engine-llama-3-1-70b-instruct/README.md rename {llama => llm/llama}/engine-llama-3-1-70b-instruct/config.yaml (95%) create mode 100644 llm/llama/engine-llama-3-1-8b-instruct/README.md rename {llama => llm/llama}/engine-llama-3-1-8b-instruct/config.yaml (95%) create mode 100644 llm/llama/engine-llama-3-3-70b-instruct/README.md rename {llama => llm/llama}/engine-llama-3-3-70b-instruct/config.yaml (95%) create mode 100644 llm/llama/engine-llama-3.1-405b-instruct/README.md rename {llama => llm/llama}/engine-llama-3.1-405b-instruct/config.yaml (93%) create mode 100644 llm/llama/llama-3-1-405b-instruct/README.md rename {llama/llama-3_1-405b-instruct => llm/llama/llama-3-1-405b-instruct}/config.yaml (84%) rename {stable-diffusion/sdxl-controlnet-canny => llm/llama/llama-3-1-405b-instruct}/model/__init__.py (100%) rename {llama/llama-3_1-405b-instruct => llm/llama/llama-3-1-405b-instruct}/model/model.py (100%) rename {llama/llama-3_1-405b-instruct => llm/llama/llama-3-1-405b-instruct}/model/sighelper.py (100%) create mode 100644 llm/llama/llama-3-1-70b-instruct/README.md rename {llama/llama-3_1_70b-instruct => llm/llama/llama-3-1-70b-instruct}/config.yaml (58%) rename {stable-diffusion/sdxl-controlnet-depth => llm/llama/llama-3-1-70b-instruct}/model/__init__.py (100%) rename {llama/llama-3_1_70b-instruct => llm/llama/llama-3-1-70b-instruct}/model/model.py (100%) rename {llama/llama-3_1_70b-instruct => llm/llama/llama-3-1-70b-instruct}/model/sighelper.py (100%) create mode 100644 llm/llama/llama-3-1-8b-instruct-sglang/README.md rename {llama/llama-3_1-8b-instruct-sglang => llm/llama/llama-3-1-8b-instruct-sglang}/config.yaml (90%) rename {stable-diffusion/sdxl-controlnet => llm/llama/llama-3-1-8b-instruct-sglang}/model/__init__.py (100%) rename {llama/llama-3_1-8b-instruct-sglang => llm/llama/llama-3-1-8b-instruct-sglang}/model/model.py (100%) create mode 100644 llm/llama/llama-3-1-8b-instruct/README.md rename {llama/llama-3_1-8b-instruct => llm/llama/llama-3-1-8b-instruct}/config.yaml (88%) rename {stable-diffusion/sdxl-lightning => llm/llama/llama-3-1-8b-instruct}/model/__init__.py (100%) rename {llama/llama-3_1-8b-instruct => llm/llama/llama-3-1-8b-instruct}/model/model.py (100%) create mode 100644 llm/llama/llama-3-2-11b-vision-instruct/README.md rename {llama/llama-3_2-11b-vision-instruct => llm/llama/llama-3-2-11b-vision-instruct}/config.yaml (95%) create mode 100644 llm/llama/llama-3-70b-instruct/README.md rename {llama => llm/llama}/llama-3-70b-instruct/config.yaml (65%) rename {stable-diffusion/sdxl-lora-swapping => llm/llama/llama-3-70b-instruct}/model/__init__.py (100%) rename {llama => llm/llama}/llama-3-70b-instruct/model/model.py (100%) create mode 100644 llm/llama/llama-3-8b-instruct/README.md rename {llama => llm/llama}/llama-3-8b-instruct/config.yaml (68%) rename {stable-diffusion/sdxl-lora => llm/llama/llama-3-8b-instruct}/model/__init__.py (100%) rename {llama => llm/llama}/llama-3-8b-instruct/model/model.py (100%) create mode 100644 llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/README.md rename {llama => llm/llama}/llama-4-maverick-17b-128e-instruct-fp8-vllm/config.yaml (88%) rename {llama => llm/llama}/llama-4-maverick-17b-128e-instruct-fp8-vllm/data/do.sh (100%) create mode 100644 llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/README.md rename {llama => llm/llama}/llama-4-scout-17b-16e-instruct-bf16-vllm/config.yaml (93%) rename {llama => llm/llama}/llama-4-scout-17b-16e-instruct-bf16-vllm/data/do.sh (100%) create mode 100644 llm/llama/tinyllama-1.1B-chat-v1.0/README.md rename {llama => llm/llama}/tinyllama-1.1B-chat-v1.0/config.yaml (85%) create mode 100644 llm/llava/llava-1.6-sgl/README.md rename {llava => llm/llava}/llava-1.6-sgl/config.yaml (54%) rename {stable-diffusion/sdxl-turbo => llm/llava/llava-1.6-sgl}/model/__init__.py (100%) rename {llava => llm/llava}/llava-1.6-sgl/model/model.py (100%) rename {llava => llm/llava}/llava-1.6-sgl/requirements.txt (100%) create mode 100644 llm/llava/llava-v1.5-7b/README.md rename {llava => llm/llava}/llava-v1.5-7b/config.yaml (63%) rename {stable-diffusion/stable-diffusion-3-medium => llm/llava/llava-v1.5-7b}/model/__init__.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/model/model.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/__init__.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/constants.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/conversation.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_gpt_review.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_bench.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_visual.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_pope.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_science_qa.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4_requery.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/eval_textvqa.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/generate_webpage_data_from_table.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/m4c_evaluator.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/model_qa.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/model_vqa.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/model_vqa_loader.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/model_vqa_mmbench.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/model_vqa_qbench.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/model_vqa_science.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/qa_baseline_gpt35.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/run_llava.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/summarize_gpt_review.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/answer/answer_alpaca-13b.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/answer/answer_bard.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/answer/answer_gpt35.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/answer/answer_llama-13b.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/answer/answer_vicuna-13b.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/caps_boxes_coco2014_val_80.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/model.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/prompt.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/question.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_13b_v0.json (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/review/review_bard_vicuna-13b.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/review/review_gpt35_vicuna-13b.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/reviewer.jsonl (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/table/rule.json (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/figures/alpaca.png (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/figures/bard.jpg (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/figures/chatgpt.svg (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/figures/llama.jpg (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/figures/vicuna.jpeg (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/index.html (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/script.js (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/eval/webpage/styles.css (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/mm_utils.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/__init__.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/apply_delta.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/builder.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/consolidate.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/llava_llama.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/llava_mpt.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/adapt_tokenizer.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/attention.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/blocks.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/configuration_mpt.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/custom_embedding.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/flash_attn_triton.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/hf_prefixlm_converter.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/meta_init_context.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/modeling_mpt.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/norm.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/language_model/mpt/param_init_fns.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/llava_arch.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/make_delta.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/multimodal_encoder/builder.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/multimodal_encoder/clip_encoder.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/multimodal_projector/builder.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/model/utils.py (100%) rename {stable-diffusion/stable-diffusion-inpainting-trt/model => llm/llava/llava-v1.5-7b/packages/llava/serve}/__init__.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/cli.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/controller.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/examples/extreme_ironing.jpg (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/examples/waterview.jpg (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/gradio_web_server.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/model_worker.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/register_worker.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/serve/test_message.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/train/llama_flash_attn_monkey_patch.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/train/llama_xformers_attn_monkey_patch.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/train/llava_trainer.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/train/train.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/train/train_mem.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/train/train_xformers.py (100%) rename {llava => llm/llava}/llava-v1.5-7b/packages/llava/utils.py (100%) create mode 100644 llm/llava/llava-v1.6-34b/README.md create mode 100644 llm/llava/llava-v1.6-34b/config.yaml rename {llava => llm/llava}/llava-v1.6-34b/input.json (100%) rename {stable-diffusion/stable-diffusion-xl-1.0-trt-h100 => llm/llava/llava-v1.6-34b}/model/__init__.py (100%) rename {llava => llm/llava}/llava-v1.6-34b/model/model.py (100%) rename {lora => llm/lora}/README.md (100%) create mode 100644 llm/lora/engine-lora/README.md rename {lora => llm/lora}/engine-lora/config.yaml (95%) create mode 100644 llm/lora/sglang-lora/README.md rename {lora => llm/lora}/sglang-lora/config.yaml (93%) create mode 100644 llm/lora/vllm-lora/README.md rename {lora => llm/lora}/vllm-lora/config.yaml (95%) create mode 100644 llm/midnight/README.md rename {midnight => llm/midnight}/config.yaml (95%) rename {stable-diffusion/stable-diffusion-xl-1.0-trt => llm/midnight}/model/__init__.py (100%) rename {midnight => llm/midnight}/model/model.py (100%) rename {midnight => llm/midnight}/requirements.txt (100%) create mode 100644 llm/minimax/minimax-m2-1/README.md rename {minimax/minimax_m2_1 => llm/minimax/minimax-m2-1}/config.yaml (96%) rename {mistral => llm/mistral/_archive}/engine-mistral-7b-instruct/README.md (100%) rename {mistral => llm/mistral/_archive}/engine-mistral-7b-instruct/config.yaml (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-chat/README.md (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-chat/config.yaml (100%) rename {stable-diffusion/stable-diffusion-xl-1.0 => llm/mistral/_archive/mistral-7b-chat}/model/__init__.py (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-chat/model/model.py (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-instruct-vllm/README.md (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-instruct-vllm/config.yaml (100%) rename {stable-diffusion/stable-diffusion => llm/mistral/_archive/mistral-7b-instruct-vllm}/model/__init__.py (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-instruct-vllm/model/model.py (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-instruct/README.md (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-instruct/config.yaml (100%) rename {stable-diffusion/stable-video-diffusion => llm/mistral/_archive/mistral-7b-instruct}/model/__init__.py (100%) rename {mistral => llm/mistral/_archive}/mistral-7b-instruct/model/model.py (100%) create mode 100644 llm/mistral/engine-devstral/README.md rename {mistral => llm/mistral}/engine-devstral/config.yaml (95%) create mode 100644 llm/mistral/engine-mistral-small-3/README.md rename {mistral => llm/mistral}/engine-mistral-small-3/config.yaml (94%) create mode 100644 llm/mistral/engine-mixtral-8x22b-instruct/README.md rename {mistral => llm/mistral}/engine-mixtral-8x22b-instruct/config.yaml (93%) create mode 100644 llm/mistral/engine-mixtral-8x7b-instruct/README.md rename {mistral => llm/mistral}/engine-mixtral-8x7b-instruct/config.yaml (94%) create mode 100644 llm/mistral/mistral-7b/README.md rename {mistral => llm/mistral}/mistral-7b/config.yaml (82%) rename {stable-diffusion/stable-video-diffusion/model/scripts => llm/mistral/mistral-7b/model}/__init__.py (100%) rename {mistral => llm/mistral}/mistral-7b/model/model.py (100%) create mode 100644 llm/mistral/mistral-small-3.1/README.md rename {mistral => llm/mistral}/mistral-small-3.1/config.yaml (91%) create mode 100644 llm/mistral/mixtral-8x22b-trt-int8-weights-only/README.md rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => llm/mistral/mixtral-8x22b-trt-int8-weights-only}/TRT-LLM-README.md (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/config.yaml (94%) rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => llm/mistral/mixtral-8x22b-trt-int8-weights-only}/data/.gitattributes (100%) rename {stable-diffusion/stable-video-diffusion/model/scripts/util => llm/mistral/mixtral-8x22b-trt-int8-weights-only/model}/__init__.py (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/model/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/packages/client.py (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/ensemble/config.pbtxt (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/config.pbtxt (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => llm/mistral/mixtral-8x22b-trt-int8-weights-only}/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-h100 => llm/mistral/mixtral-8x22b-trt-int8-weights-only}/packages/utils.py (100%) create mode 100644 llm/mistral/mixtral-8x22b/README.md rename {mistral => llm/mistral}/mixtral-8x22b/config.yaml (86%) rename {stable-diffusion/stable-video-diffusion/model/scripts/util/detection => llm/mistral/mixtral-8x22b/model}/__init__.py (100%) rename {mistral => llm/mistral}/mixtral-8x22b/model/model.py (100%) create mode 100644 llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/TRT-LLM-README.md (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-h100/config.yaml (94%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/data/.gitattributes (100%) rename {templates/faster-whisper-truss => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/model/__init__.py (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/model/model.py (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/packages/client.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/packages/inflight_batcher_llm/postprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/packages/inflight_batcher_llm/preprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 => llm/mistral/mixtral-8x7b-instruct-trt-llm-h100}/packages/utils.py (100%) create mode 100644 llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/TRT-LLM-README.md (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/config.yaml (94%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/data/.gitattributes (100%) rename {templates/transformers-openai-compatible => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/model/__init__.py (100%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/model/model.py (100%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/packages/client.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/packages/inflight_batcher_llm/postprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/packages/inflight_batcher_llm/preprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100}/packages/utils.py (100%) create mode 100644 llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md rename {mistral/mixtral-8x7b-instruct-trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant}/TRT-LLM-README.md (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml (98%) rename {mistral/mixtral-8x7b-instruct-trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant}/data/.gitattributes (100%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant}/model/__init__.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/client.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/ensemble/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant}/packages/inflight_batcher_llm/postprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant}/packages/inflight_batcher_llm/preprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant}/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt (100%) rename {mistral/mixtral-8x7b-instruct-trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant}/packages/utils.py (100%) create mode 100644 llm/mistral/mixtral-8x7b-instruct-trt-llm/README.md rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm}/TRT-LLM-README.md (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm/config.yaml (94%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm}/data/.gitattributes (100%) rename {trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput => llm/mistral/mixtral-8x7b-instruct-trt-llm}/model/__init__.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm/model/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm/packages/client.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt (100%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm}/packages/inflight_batcher_llm/postprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt (100%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm}/packages/inflight_batcher_llm/preprocessing/1/model.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt (100%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm}/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt (100%) rename {templates/trt-llm => llm/mistral/mixtral-8x7b-instruct-trt-llm}/packages/utils.py (100%) create mode 100644 llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md rename {mistral => llm/mistral}/mixtral-8x7b-instruct-vllm-a100-t-tp2/config.yaml (53%) rename {trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context => llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2}/model/__init__.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/model.py (100%) create mode 100644 llm/mistral/mixtral-8x7b-instruct-vllm/README.md rename {mistral => llm/mistral}/mixtral-8x7b-instruct-vllm/config.yaml (57%) rename {trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft => llm/mistral/mixtral-8x7b-instruct-vllm}/model/__init__.py (100%) rename {mistral => llm/mistral}/mixtral-8x7b-instruct-vllm/model/model.py (100%) create mode 100644 llm/mistral/pixtral-12b/README.md rename {mistral => llm/mistral}/pixtral-12b/config.yaml (94%) rename {trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput => llm/mistral/pixtral-12b}/model/__init__.py (100%) rename {mistral => llm/mistral}/pixtral-12b/model/model.py (100%) create mode 100644 llm/nemotron/llama-3-1-nemotron-70b-instruct/README.md rename {nemotron => llm/nemotron}/llama-3-1-nemotron-70b-instruct/config.yaml (90%) create mode 100644 llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1/README.md rename {nemotron/Llama-3-1-Nemotron-Nano-VL-8B-V1 => llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1}/config.yaml (88%) create mode 100644 llm/nemotron/llama-nemoretriever-colembed-3b-v1/README.md rename {nemotron => llm/nemotron}/llama-nemoretriever-colembed-3b-v1/base_image/Dockerfile (100%) rename {nemotron => llm/nemotron}/llama-nemoretriever-colembed-3b-v1/config.yaml (100%) rename {nemotron => llm/nemotron}/llama-nemoretriever-colembed-3b-v1/model/model.py (100%) create mode 100644 llm/nemotron/nemotron-3-nano-nvfp4/README.md rename {nemotron/Nemotron-3-Nano-NVFP4 => llm/nemotron/nemotron-3-nano-nvfp4}/config.yaml (93%) create mode 100644 llm/nemotron/nemotron-3-nano/README.md rename {nemotron/Nemotron-3-Nano => llm/nemotron/nemotron-3-nano}/config.yaml (93%) create mode 100644 llm/nemotron/nemotron-nano-12b-v2-vl-bf16/README.md rename {nemotron/Nemotron-Nano-12B-v2-VL-BF16 => llm/nemotron/nemotron-nano-12b-v2-vl-bf16}/config.yaml (88%) create mode 100644 llm/nemotron/nemotron-ultra-253b/README.md rename {nemotron => llm/nemotron}/nemotron-ultra-253b/config.yaml (91%) create mode 100644 llm/nsql/README.md rename {nsql => llm/nsql}/config.yaml (73%) rename {trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context => llm/nsql}/model/__init__.py (100%) rename {nsql => llm/nsql}/model/model.py (100%) create mode 100644 llm/openai/gpt-oss-120b/README.md rename {openai => llm/openai}/gpt-oss-120b/config.yaml (97%) create mode 100644 llm/openai/gpt-oss-20b/README.md rename {openai => llm/openai}/gpt-oss-20b/config.yaml (97%) rename {personaplex-7b-v1 => llm/personaplex-7b-v1}/Dockerfile (100%) create mode 100644 llm/personaplex-7b-v1/README.md rename {personaplex-7b-v1 => llm/personaplex-7b-v1}/client.py (100%) rename {personaplex-7b-v1 => llm/personaplex-7b-v1}/config.yaml (90%) rename {personaplex-7b-v1 => llm/personaplex-7b-v1}/requirements-client.txt (100%) create mode 100644 llm/phi/phi-3-mini-128k-instruct/README.md create mode 100644 llm/phi/phi-3-mini-128k-instruct/config.yaml rename {trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft => llm/phi/phi-3-mini-128k-instruct}/model/__init__.py (100%) rename {phi => llm/phi}/phi-3-mini-128k-instruct/model/model.py (100%) create mode 100644 llm/phi/phi-3-mini-4k-instruct/README.md create mode 100644 llm/phi/phi-3-mini-4k-instruct/config.yaml rename {ultravox => llm/phi/phi-3-mini-4k-instruct}/model/__init__.py (100%) rename {phi => llm/phi}/phi-3-mini-4k-instruct/model/model.py (100%) create mode 100644 llm/phi/phi-3.5-mini/README.md rename {phi => llm/phi}/phi-3.5-mini/config.yaml (87%) rename {vllm => llm/phi/phi-3.5-mini}/model/__init__.py (100%) rename {vllm => llm/phi/phi-3.5-mini}/model/helper.py (100%) rename {vllm => llm/phi/phi-3.5-mini}/model/model.py (100%) rename {qwen => llm/qwen/_archive}/qwen-7b-chat/README.md (100%) rename {qwen => llm/qwen/_archive}/qwen-7b-chat/config.yaml (100%) rename {whisper/faster-whisper-small => llm/qwen/_archive/qwen-7b-chat}/model/__init__.py (100%) rename {qwen => llm/qwen/_archive}/qwen-7b-chat/model/model.py (100%) create mode 100644 llm/qwen/engine-qwen-2-5-14b-coder-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-14b-coder-instruct/config.yaml (94%) create mode 100644 llm/qwen/engine-qwen-2-5-14b-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-14b-instruct/config.yaml (94%) create mode 100644 llm/qwen/engine-qwen-2-5-32b-coder-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-32b-coder-instruct/config.yaml (94%) create mode 100644 llm/qwen/engine-qwen-2-5-32b-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-32b-instruct/config.yaml (95%) create mode 100644 llm/qwen/engine-qwen-2-5-3b-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-3b-instruct/config.yaml (95%) create mode 100644 llm/qwen/engine-qwen-2-5-72b-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-72b-instruct/config.yaml (95%) create mode 100644 llm/qwen/engine-qwen-2-5-72b-math-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-72b-math-instruct/config.yaml (94%) create mode 100644 llm/qwen/engine-qwen-2-5-7b-coder-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-7b-coder-instruct/config.yaml (95%) create mode 100644 llm/qwen/engine-qwen-2-5-7b-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-7b-instruct/config.yaml (95%) create mode 100644 llm/qwen/engine-qwen-2-5-7b-math-instruct/README.md rename {qwen => llm/qwen}/engine-qwen-2-5-7b-math-instruct/config.yaml (94%) create mode 100644 llm/qwen/engine-qwen-3-06b/README.md rename {qwen => llm/qwen}/engine-qwen-3-06b/config.yaml (95%) create mode 100644 llm/qwen/engine-qwen-3-30b-a3b-instruct-2507/README.md rename {qwen => llm/qwen}/engine-qwen-3-30b-a3b-instruct-2507/config.yaml (92%) create mode 100644 llm/qwen/engine-qwen-3-32b/README.md rename {qwen => llm/qwen}/engine-qwen-3-32b/config.yaml (95%) create mode 100644 llm/qwen/engine-qwen-3-4b/README.md rename {qwen => llm/qwen}/engine-qwen-3-4b/config.yaml (95%) rename {qwen => llm/qwen}/model_auto.py (100%) create mode 100644 llm/qwen/qwen-3-235B-A22B-instruct-2507-trt/README.md rename {qwen => llm/qwen}/qwen-3-235B-A22B-instruct-2507-trt/config.yaml (95%) create mode 100644 llm/qwen/qwen-3-235B-sglang/README.md rename {qwen => llm/qwen}/qwen-3-235B-sglang/config.yaml (94%) create mode 100644 llm/qwen/qwen-3-30B-A3-coder/README.md rename {qwen => llm/qwen}/qwen-3-30B-A3-coder/config.yaml (94%) create mode 100644 llm/qwen/qwen-3-30B-A3-sglang/README.md rename {qwen => llm/qwen}/qwen-3-30B-A3-sglang/config.yaml (95%) create mode 100644 llm/qwen/qwen-3-30B-A3-vllm/README.md rename {qwen => llm/qwen}/qwen-3-30B-A3-vllm/config.yaml (93%) create mode 100644 llm/qwen/qwen-3-32B-sglang/README.md rename {qwen => llm/qwen}/qwen-3-32B-sglang/config.yaml (95%) create mode 100644 llm/qwen/qwen-3-next-80B-A3-instruct-sglang/README.md rename {qwen => llm/qwen}/qwen-3-next-80B-A3-instruct-sglang/config.yaml (96%) create mode 100644 llm/qwen/qwen-3-next-80B-A3-thinking-sglang/README.md rename {qwen => llm/qwen}/qwen-3-next-80B-A3-thinking-sglang/config.yaml (95%) create mode 100644 llm/qwen/qwen-3-vl-30b-a3b-instruct/README.md rename {qwen => llm/qwen}/qwen-3-vl-30b-a3b-instruct/config.yaml (92%) create mode 100644 llm/qwen/qwen-3-vl-30b-a3b-thinking/README.md rename {qwen => llm/qwen}/qwen-3-vl-30b-a3b-thinking/config.yaml (92%) create mode 100644 llm/qwen/qwen-3-vl-32b/README.md rename {qwen => llm/qwen}/qwen-3-vl-32b/config.yaml (87%) create mode 100644 llm/qwen/qwen-coder-next/README.md rename {qwen => llm/qwen}/qwen-coder-next/config.yaml (92%) create mode 100644 llm/qwen/qwen-image/README.md rename {qwen => llm/qwen}/qwen-image/assets/generated_image.jpg (100%) rename {qwen => llm/qwen}/qwen-image/config.yaml (84%) rename {qwen => llm/qwen}/qwen-image/model/model.py (100%) create mode 100644 llm/qwen/qwen-vl/README.md rename {qwen => llm/qwen}/qwen-vl/config.yaml (69%) rename {whisper/faster-whisper-v2 => llm/qwen/qwen-vl}/model/__init__.py (100%) rename {qwen => llm/qwen}/qwen-vl/model/model.py (100%) create mode 100644 llm/seed/seed-llm/README.md rename {seed/seed_llm => llm/seed/seed-llm}/config.yaml (95%) create mode 100644 llm/z-ai/glm-4-5-air-fp8/README.md rename {z-ai => llm/z-ai}/glm-4-5-air-fp8/config.yaml (91%) create mode 100644 llm/z-ai/glm-4-5-fp8/README.md rename {z-ai => llm/z-ai}/glm-4-5-fp8/config.yaml (92%) create mode 100644 llm/z-ai/glm-4-5-v/README.md rename {z-ai/glm_4_5_v => llm/z-ai/glm-4-5-v}/config.yaml (92%) create mode 100644 llm/z-ai/glm-4-6-fp8/README.md rename {z-ai => llm/z-ai}/glm-4-6-fp8/config.yaml (93%) create mode 100644 llm/z-ai/glm-4-7-flash/README.md rename {z-ai/glm_4_7_flash => llm/z-ai/glm-4-7-flash}/config.yaml (92%) delete mode 100644 lora/engine-lora/README.md delete mode 100644 lora/sglang-lora/README.md delete mode 100644 lora/vllm-lora/README.md delete mode 100644 magic-animate/README.md delete mode 100644 metavoice-1b/README.md delete mode 100644 midnight/README.md delete mode 100644 mistral/engine-mistral-small-3/README.md delete mode 100644 mistral/engine-mixtral-8x22b-instruct/README.md delete mode 100644 mistral/engine-mixtral-8x7b-instruct/README.md delete mode 100644 mistral/mistral-7b/README.md delete mode 100644 mistral/mixtral-8x22b-trt-int8-weights-only/README.md delete mode 100644 mistral/mixtral-8x22b/README.md delete mode 100644 mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md delete mode 100644 mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md delete mode 100644 mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md delete mode 100644 mistral/mixtral-8x7b-instruct-trt-llm/README.md delete mode 100644 mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md delete mode 100644 mistral/mixtral-8x7b-instruct-vllm/README.md delete mode 100644 mistral/pixtral-12b/README.md delete mode 100644 multiprocessing/README.md delete mode 100644 musicgen-large/README.md delete mode 100644 musicgen-melody/README.md delete mode 100644 nemotron/llama-3-1-nemotron-70b-instruct/README.md delete mode 100644 nemotron/llama-nemoretriever-colembed-3b-v1/README.md delete mode 100644 nsql/README.md delete mode 100644 openai/gpt-oss-120b/README.md delete mode 100644 openai/gpt-oss-20b/README.md create mode 100644 optimized/README.md create mode 100644 optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4 => optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4}/config.yaml (89%) create mode 100644 optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only => optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only}/config.yaml (90%) create mode 100644 optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8 => optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8}/config.yaml (90%) create mode 100644 optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4 => optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4}/config.yaml (90%) create mode 100644 optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4 => optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4}/config.yaml (90%) create mode 100644 optimized/bisv2/nvidia-qwen3-30b-a3b-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4 => optimized/bisv2/nvidia-qwen3-30b-a3b-fp4}/config.yaml (90%) create mode 100644 optimized/bisv2/nvidia-qwen3-8b-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4 => optimized/bisv2/nvidia-qwen3-8b-fp4}/config.yaml (91%) create mode 100644 optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4 => optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4}/config.yaml (91%) create mode 100644 optimized/bisv2/qwen-qwen2.5-coder-7b-instruct/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct => optimized/bisv2/qwen-qwen2.5-coder-7b-instruct}/config.yaml (91%) create mode 100644 optimized/bisv2/qwen-qwen3-32b-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4 => optimized/bisv2/qwen-qwen3-32b-fp4}/config.yaml (91%) create mode 100644 optimized/bisv2/qwen-qwen3-4b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8 => optimized/bisv2/qwen-qwen3-4b-fp8}/config.yaml (91%) create mode 100644 optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8 => optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8}/config.yaml (91%) create mode 100644 optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8 => optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8}/config.yaml (92%) create mode 100644 optimized/briton/google-gemma-3-1b-it/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it => optimized/briton/google-gemma-3-1b-it}/config.yaml (92%) create mode 100644 optimized/briton/google-gemma-3-270m-it/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it => optimized/briton/google-gemma-3-270m-it}/config.yaml (92%) create mode 100644 optimized/briton/google-gemma-3-27b-it-speculative-lookahead/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead => optimized/briton/google-gemma-3-27b-it-speculative-lookahead}/config.yaml (93%) create mode 100644 optimized/briton/google-gemma-3-27b-it/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it => optimized/briton/google-gemma-3-27b-it}/config.yaml (91%) create mode 100644 optimized/briton/meta-llama-llama-3.1-405b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8 => optimized/briton/meta-llama-llama-3.1-405b-fp8}/config.yaml (92%) create mode 100644 optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8 => optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8}/config.yaml (94%) create mode 100644 optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8 => optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8}/config.yaml (92%) create mode 100644 optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8 => optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8}/config.yaml (93%) create mode 100644 optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8 => optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8}/config.yaml (92%) create mode 100644 optimized/briton/meta-llama-llama-3.2-3b-instruct/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct => optimized/briton/meta-llama-llama-3.2-3b-instruct}/config.yaml (91%) create mode 100644 optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4 => optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4}/config.yaml (91%) create mode 100644 optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8 => optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8}/config.yaml (92%) create mode 100644 optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8 => optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8}/config.yaml (92%) create mode 100644 optimized/briton/microsoft-phi-4-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8 => optimized/briton/microsoft-phi-4-fp8}/config.yaml (93%) create mode 100644 optimized/briton/mistralai-mistral-7b-instruct-v0.3/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3 => optimized/briton/mistralai-mistral-7b-instruct-v0.3}/config.yaml (91%) create mode 100644 optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8 => optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8}/config.yaml (91%) create mode 100644 optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8 => optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8}/config.yaml (92%) create mode 100644 optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8 => optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8}/config.yaml (94%) create mode 100644 optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only => optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only}/config.yaml (92%) create mode 100644 optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8 => optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8}/config.yaml (94%) create mode 100644 optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8 => optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8}/config.yaml (93%) create mode 100644 optimized/briton/qwen-qwen3-30b-a3b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8 => optimized/briton/qwen-qwen3-30b-a3b-fp8}/config.yaml (93%) create mode 100644 optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8 => optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8}/config.yaml (91%) create mode 100644 optimized/briton/qwen-qwen3-30b-a3b/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b => optimized/briton/qwen-qwen3-30b-a3b}/config.yaml (92%) create mode 100644 optimized/briton/qwen-qwen3-32b-fp4-mlp-only/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only => optimized/briton/qwen-qwen3-32b-fp4-mlp-only}/config.yaml (93%) create mode 100644 optimized/briton/qwen-qwen3-32b-fp4/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4 => optimized/briton/qwen-qwen3-32b-fp4}/config.yaml (93%) create mode 100644 optimized/briton/qwen-qwen3-32b-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8 => optimized/briton/qwen-qwen3-32b-fp8}/config.yaml (93%) create mode 100644 optimized/briton/qwen-qwen3-8b-min-latency-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8 => optimized/briton/qwen-qwen3-8b-min-latency-fp8}/config.yaml (95%) create mode 100644 optimized/briton/qwen-qwq-32b-reasoning-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8 => optimized/briton/qwen-qwq-32b-reasoning-fp8}/config.yaml (93%) create mode 100644 optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8 => optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8}/config.yaml (95%) create mode 100644 optimized/briton/tiiuae-falcon3-10b-instruct-fp8/README.md rename {11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8 => optimized/briton/tiiuae-falcon3-10b-instruct-fp8}/config.yaml (92%) delete mode 100644 orpheus-best-performance/README.md delete mode 100644 personaplex-7b-v1/README.md delete mode 100644 phi/phi-3-mini-128k-instruct/README.md delete mode 100644 phi/phi-3-mini-128k-instruct/config.yaml delete mode 100644 phi/phi-3-mini-4k-instruct/README.md delete mode 100644 phi/phi-3-mini-4k-instruct/config.yaml delete mode 100644 phi/phi-3.5-mini/README.md delete mode 100644 piper-tts/README.md delete mode 100644 playground-v2-aesthetic/README.md delete mode 100644 poetry.lock delete mode 100644 pyproject.toml delete mode 100644 qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/README.md delete mode 100644 qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/config.yaml delete mode 100644 qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/README.md delete mode 100644 qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/config.yaml delete mode 100644 qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md delete mode 100644 qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/config.yaml delete mode 100644 qwen/BEI-qwen-qwen3-embedding-4b-fp8/README.md delete mode 100644 qwen/BEI-qwen-qwen3-embedding-4b-fp8/config.yaml delete mode 100644 qwen/BEI-qwen-qwen3-embedding-8b-fp8/README.md delete mode 100644 qwen/BEI-qwen-qwen3-embedding-8b-fp8/config.yaml delete mode 100644 qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md delete mode 100644 qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/config.yaml delete mode 100644 qwen/BEI-qwen-qwen3-reranker-4b-fp8/README.md delete mode 100644 qwen/BEI-qwen-qwen3-reranker-4b-fp8/config.yaml delete mode 100644 qwen/BEI-qwen-qwen3-reranker-8b-fp8/README.md delete mode 100644 qwen/BEI-qwen-qwen3-reranker-8b-fp8/config.yaml delete mode 100644 qwen/engine-qwen-2-5-14b-coder-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-14b-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-32b-coder-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-32b-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-3b-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-72b-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-72b-math-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-7b-coder-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-7b-instruct/README.md delete mode 100644 qwen/engine-qwen-2-5-7b-math-instruct/README.md delete mode 100644 qwen/qwen-3-30B-A3-coder/README.md delete mode 100644 qwen/qwen-3-30b-omni-thinker/README.md delete mode 100644 qwen/qwen-3-30b-omni/README.md delete mode 100644 qwen/qwen-3-asr/README.md delete mode 100644 qwen/qwen-image/README.md delete mode 100644 qwen/qwen-vl/README.md delete mode 100644 segment-anything/README.md delete mode 100644 stable-diffusion/dreamshaper-lcm/README.md delete mode 100644 stable-diffusion/dreamshaper-lcm/config.yaml delete mode 100644 stable-diffusion/playground-v2-trt/README.md delete mode 100644 stable-diffusion/sd-textual-inversion/README.md delete mode 100644 stable-diffusion/sd-turbo/README.md delete mode 100644 stable-diffusion/sdxl-controlnet-canny/README.md delete mode 100644 stable-diffusion/sdxl-controlnet-depth/README.md delete mode 100644 stable-diffusion/sdxl-controlnet/README.md delete mode 100644 stable-diffusion/sdxl-lightning/README.md delete mode 100644 stable-diffusion/sdxl-lora-swapping/README.md delete mode 100644 stable-diffusion/sdxl-lora/README.md delete mode 100644 stable-diffusion/sdxl-lora/config.yaml delete mode 100644 stable-diffusion/sdxl-turbo/README.md delete mode 100644 stable-diffusion/stable-diffusion-3-medium/README.md delete mode 100644 stable-diffusion/stable-diffusion-inpainting-trt/README.md delete mode 100644 stable-diffusion/stable-diffusion-inpainting-trt/config.yaml delete mode 100644 stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md delete mode 100644 stable-diffusion/stable-diffusion-xl-1.0-trt/README.md delete mode 100644 stable-diffusion/stable-diffusion-xl-1.0/README.md delete mode 100644 stable-diffusion/stable-diffusion/README.md delete mode 100644 stable-diffusion/stable-video-diffusion/README.md delete mode 100644 text-embeddings-inference/README.md create mode 100644 tutorials/README.md create mode 100644 tutorials/cached-weights/README.md rename {06-high-performance-cached-weights => tutorials/cached-weights}/config.yaml (94%) rename {06-high-performance-cached-weights => tutorials/cached-weights}/doc.yaml (100%) rename {whisper/faster-whisper-v3 => tutorials/cached-weights}/model/__init__.py (100%) rename {06-high-performance-cached-weights => tutorials/cached-weights}/model/model.py (100%) rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/.gitignore (100%) rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/.truss_ignore (100%) create mode 100644 tutorials/dynamic-batching/README.md rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/config.yaml (82%) rename {whisper/whisper-streaming => tutorials/dynamic-batching}/model/__init__.py (100%) rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/model/model.py (100%) rename {whisper/whisper-torchserve/model => tutorials/dynamic-batching/packages}/__init__.py (100%) rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/packages/run.py (100%) rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/packages/tokenizer.py (100%) rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/packages/whisper_utils.py (100%) rename {07-high-performance-dynamic-batching => tutorials/dynamic-batching}/test.py (100%) create mode 100644 tutorials/getting-started-bert/README.md rename {01-getting-started-bert => tutorials/getting-started-bert}/config.yaml (92%) rename {01-getting-started-bert => tutorials/getting-started-bert}/doc.yaml (100%) rename {whisper/whisper-truss => tutorials/getting-started-bert}/model/__init__.py (100%) rename {01-getting-started-bert => tutorials/getting-started-bert}/model/model.py (100%) create mode 100644 tutorials/image-generation/README.md rename {04-image-generation => tutorials/image-generation}/config.yaml (94%) rename {04-image-generation => tutorials/image-generation}/doc.yaml (100%) rename {whisper/whisper-v3-truss-base64 => tutorials/image-generation}/model/__init__.py (100%) rename {04-image-generation => tutorials/image-generation}/model/model.py (100%) create mode 100644 tutorials/llm-basics/README.md rename {02-llm => tutorials/llm-basics}/config.yaml (88%) rename {02-llm => tutorials/llm-basics}/doc.yaml (100%) rename {whisper/whisper-v3-truss => tutorials/llm-basics}/model/__init__.py (100%) rename {02-llm => tutorials/llm-basics}/model/model.py (100%) create mode 100644 tutorials/llm-streaming/README.md rename {03-llm-with-streaming => tutorials/llm-streaming}/config.yaml (81%) rename {03-llm-with-streaming => tutorials/llm-streaming}/doc.yaml (100%) rename {whisper/whisperx-truss => tutorials/llm-streaming}/model/__init__.py (100%) rename {03-llm-with-streaming => tutorials/llm-streaming}/model/model.py (100%) create mode 100644 tutorials/private-huggingface/README.md rename {09-private-huggingface => tutorials/private-huggingface}/config.yaml (83%) rename {09-private-huggingface => tutorials/private-huggingface}/doc.yaml (100%) rename {xtts-streaming => tutorials/private-huggingface}/model/__init__.py (100%) rename {09-private-huggingface => tutorials/private-huggingface}/model/model.py (100%) create mode 100644 tutorials/speech-to-text/README.md rename {05-speech-to-text => tutorials/speech-to-text}/config.yaml (81%) rename {xtts-v2-truss => tutorials/speech-to-text}/model/__init__.py (100%) rename {05-speech-to-text => tutorials/speech-to-text}/model/model.py (100%) create mode 100644 tutorials/system-packages/README.md rename {10-using-system-packages => tutorials/system-packages}/config.yaml (90%) rename {10-using-system-packages => tutorials/system-packages}/doc.yaml (100%) rename whisper/whisper-truss/data/.gitkeep => tutorials/system-packages/model/__init__.py (100%) rename {10-using-system-packages => tutorials/system-packages}/model/model.py (100%) delete mode 100644 ultravox/README.md delete mode 100644 whisper/faster-whisper-small/README.md delete mode 100644 whisper/whisper-streaming/README.md delete mode 100644 whisper/whisper-truss/README.md delete mode 100644 whisper/whisper-v3-truss-base64/README.md delete mode 100644 whisper/whisper-v3-truss/README.md delete mode 100644 whisper/whisperx-truss/README.md delete mode 100644 xtts-streaming/README.md delete mode 100644 xtts-v2-truss/README.md delete mode 100644 z-ai/glm-4-5-air-fp8/README.md delete mode 100644 z-ai/glm-4-5-fp8/README.md diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md new file mode 100644 index 000000000..d33dbe517 --- /dev/null +++ b/.claude/CLAUDE.md @@ -0,0 +1,127 @@ +# Truss Examples β€” Project Rules + +This repo contains production-ready inference examples for Truss. Follow these rules when creating, editing, or reviewing examples. + +## Repository structure + +Top-level directories: + +- `tutorials/` β€” Getting-started guides (BERT, LLMs, streaming, image generation, caching, batching) +- `llm/` β€” Text generation models, organized by family (`llm/llama/`, `llm/qwen/`, `llm/mistral/`, etc.) +- `embeddings/` β€” Embedding and reranker models (`bei/`, `tei/`, `clip/`) +- `image/` β€” Image generation, editing, segmentation (`stable-diffusion/`, `flux/`, `segment-anything/`) +- `audio/` β€” Speech-to-text, TTS, music generation (`whisper/`, `kokoro/`, `musicgen-large/`) +- `optimized/` β€” Autogenerated TRT-LLM configs (`briton/`, `bisv2/`) +- `infrastructure/` β€” Patterns and techniques (custom servers, gRPC, model caching, chains) +- `_internal/` β€” Build tooling, templates, test scripts (not customer-facing) +- `_archive/` β€” Deprecated examples + +## Placement rules + +| Model type | Directory | Example | +|---|---|---| +| Text generation / chat / LLM | `llm//` | `llm/llama/tinyllama-1.1B-chat-v1.0` | +| Embedding or reranker | `embeddings/bei/`, `embeddings/tei/`, or `embeddings/clip/` | `embeddings/bei/baai-bge-large-en-v1.5-embedding` | +| Image generation or processing | `image/` or `image//` | `image/stable-diffusion/stable-diffusion-xl-1.0` | +| Audio (STT, TTS, music) | `audio/` | `audio/whisper/faster-whisper-v3` | +| Infrastructure pattern | `infrastructure/` | `infrastructure/model-cache` | +| Getting-started tutorial | `tutorials/` | `tutorials/getting-started-bert` | + +## Naming conventions + +- Use hyphens, not underscores: `falcon-7b` not `falcon_7b` +- Do not include "truss" in folder names: `falcon-7b` not `falcon-7b-truss` +- Include parameter count when multiple variants exist: `Falcon 7B` not `Falcon` +- All directory names should be lowercase + +## config.yaml requirements + +Always include: +- `model_name` +- `description` (one-sentence summary) +- `model_metadata.example_model_input` + +Recommended: +- `model_metadata.repo_id` (Hugging Face model ID) +- `model_metadata.avatar_url` (128x128 PNG) +- `model_metadata.cover_image_url` (452x423 PNG) +- `model_metadata.tags` + +### Requirements pinning + +Pin versions for ALL Python requirements: + +```yaml +requirements: +- accelerate==0.20.3 +- torch==2.0.1 +- transformers==4.30.2 +``` + +Never leave a requirement unpinned. For git+ dependencies, pin to a specific commit hash, not `@master` or `@main`. + +### Secrets + +If the model requires a HuggingFace token, always name the secret `hf_access_token`: + +```yaml +secrets: + hf_access_token: "ENTER HF ACCESS TOKEN HERE" +``` + +### Hardware + +Configure with the least expensive hardware that runs at reasonable speed and quality. Note tradeoffs in the README when applicable. + +## README template + +Every example must include a `README.md` following this structure: + +```markdown +# + + + +## Deploying + +`.> + +## Invoking + + +``` + +- Deploy path must match the actual directory path relative to repo root +- OpenAI-compatible models must show `/v1/chat/completions` endpoint +- Never use `--trusted` or `--publish` flags +- If the config requires `hf_access_token`, the README must mention setting up the secret + +Reference example: `image/stable-diffusion/stable-diffusion-xl-1.0` + +## Model I/O conventions + +- Models that support streaming should accept a `stream` kwarg defaulting to false +- Models that take text input should call the parameter `prompt` + +## CI + +Active examples should be listed in `ci.yaml` at the repo root. The test suite (`_internal/bin/test_all.py`) validates all configs, READMEs, naming, links, pinning, and CI completeness. + +Run tests locally: +```bash +python _internal/bin/test_all.py # default +python _internal/bin/test_all.py --verbose # show every check +python _internal/bin/test_all.py --category llm # filter by category +``` + +## Automatic documentation + +To include an example in auto-generated docs on https://truss.baseten.co/, add a `doc.yaml`: + +```yaml +title: "Text-to-image" +description: "Building a text-to-image model with SDXL" +files: + - model/model.py + - config.yaml +``` diff --git a/.droid.yaml b/.droid.yaml deleted file mode 100644 index ab24dfd50..000000000 --- a/.droid.yaml +++ /dev/null @@ -1,62 +0,0 @@ -code: - plan_guidelines: - - These models are meant to be quick start examples. We are always generating a new example. Always include all necessary files. - - Never modify the README at the root of the repo. - - Always create the model first. - - Always create the requirements.txt file before the config. Point to the requirements.txt file in the config. - - Instead of putting the python requirements in the `config.yaml` file, we recommend putting them in a `requirements.txt` file. Then in the config.yaml file, you can use the `requirements_file` field to point to the `requirements.txt` file. This will make it easier to manage the dependencies and make it easier for the users to install the dependencies. - - The config creation step will have full access to the documentation for the config file. We will determine which specific keys to use at that point - - Model caching should be done in the config file. You do not need to handle this in the model. - - Always create the README for the model last. The README should always be markdown. - coding_tips: - - path : "**/config.yaml" - instructions: | - - The reference docs for the config file are at: https://truss.baseten.co/reference/config - DO NOT use values that are not in the reference docs. - - Add a model_metadata section to the config. Inside the model_metadata include a value for example_model_input - - example_model_input should be a JSON string that can be passed to the model's predict function to test its functionality. - - If a base64 image is required simply leave the string as 'image'. For example you might do "{'image': 'image'}" as an example input to a model that takes a single image as input. - - We will be using this example input to test the model. Ensure that the JSON is well formmatted and conforms to the expected inputs of the predict() function in model.py. - - make sure to include ALL dependencies. - - Always use py311 python_version. - - Default to using an A10G for the GPU when relevant. This should go under the resources section. - - Look at the model.py file closely and ensure that all the necessary information is included. For example if the model requires a GPU specify that a GPU is required in the config file. - - Any caching of models that's required should happen here in the config. - - path: "**/README.md" - instructions: | - - The readme should always credit the creators of the model and the dataset. It should also include a link to the original source of the model and the dataset. - - The explanation for how to run the model should use Baseten's platform as an example. Look at the existing READMEs in the repo. - - The README should be markdown formatted. - - The structure of the READMEs should be as follows. Do not deviate from this format: - - Truss - - A short introduction to the model and its capabilities. Include any acknowledgements here. - - Deploying on Baseten - - Instructions for cloning the repo (git clone https://github.com/basetenlabs/truss-examples/). - - And deploying the model on Baseten using `truss push`. - - Input - - The format of the input. All inputs are json. Specify if the model expects a base64 encoded image or a string, or a dictionary with multiple value. - - Explain what each input does. - - Output - - The format of the output. All outputs are json. Specify if the model returns a base64 encoded image or a string, or a dictionary with multiple value. - - Example usage - - Instructions for using the model on Baseten using `truss predict`. - - Be specific about input parameters and the expected output. - - e.g. `truss predict --input '{"prompt": "A photo of a cat"}'` - - Make it relevant to the model. - - The truss will never be executed locally. The README should not include instructions for running the model locally. - - path: "**/requirements.txt" - instructions: | - - Always pin the dependencies in the requirements using >=. - - Some dependencies may require you to install directly from a git repository. In that case, you can specify it like 'git+.git' - - path: "**/model.py" - instructions: | - - These models are being created as examples for the Truss platform. They should be runnable without additional configuration. That means you should select reasonable default values and not leave decisions up to the user. - - For image models that use a pretrained base model always use SG161222/RealVisXL_V3.0 - - If there is an example of how to use the model in an .ipnyb file, use it. Use as much of it as possible. For the load() method, et up the pipelines as they have done so in the notebooks. Implement predict() to run prediction as they have done in the notebooks. - - All models must implement a load() function and a predict() function. The load() function should load the model and the predict() function should take in a dict of the input parameters and return the output. For image inputs and outputs use base64 strings. - - Images passed as inputs will be base64 encoded strings. Make sure to properly process these inputs into a format that's usable by the model. - - The output of the predict function will be converted to JSON and sent over the internet. Make sure that the output is JSON serializable. For images convert them to base64 strings. - - The model class must be called Model. - - Do not make up functions. Only use functions that you know exist or you can see being used in example code. - - Ensure all your imports are correct. Use the examples provided to ensure you're importing things correctly. - - When creating prompt templates, ensure there is enough detail filled in for a user to understand the model's capabilities and how to use it. If there are any specific requirements for the prompt, ensure they are included in the prompt. - - Try to always use a GPU if the model supports it. diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml deleted file mode 100644 index e02b0b6ca..000000000 --- a/.github/actions/setup-python/action.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Setup Python" -description: "setups python, poetry and associated cache" - - -runs: - using: "composite" - steps: - - uses: actions/setup-python@v5 - with: - python-version: '3.11.4' - - - name: Get full Python version - id: full-python-version - shell: bash - run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - - run: curl -sSL https://install.python-poetry.org | python3 - - shell: bash - - - name: Configure poetry - run: poetry config virtualenvs.in-project true - shell: bash - - - name: Set up cache - uses: actions/cache@v4 - id: cache - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Ensure cache is healthy - if: steps.cache.outputs.cache-hit == 'true' - run: timeout 10s poetry run pip --version || rm -rf .venv - shell: bash diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 85e832de1..a7189709b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-python/ - - run: poetry install - - run: poetry run pre-commit run --all-files + - uses: actions/setup-python@v5 + with: + python-version: '3.11.4' + - run: pip install truss pre-commit + - run: pre-commit run --all-files diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index a277d58af..ddabb5436 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -7,6 +7,22 @@ on: workflow_dispatch: jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python environment + uses: actions/setup-python@v5 + with: + python-version: '3.11.4' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install git+https://github.com/basetenlabs/truss.git pyyaml --upgrade + - name: Run local validation (test_all.py) + run: | + python _internal/bin/test_all.py + generate_tests: runs-on: ubuntu-latest outputs: @@ -25,6 +41,7 @@ jobs: ci: runs-on: ubuntu-latest needs: + - validate - generate_tests strategy: fail-fast: false @@ -43,7 +60,7 @@ jobs: pip install git+https://github.com/basetenlabs/truss.git requests tenacity --upgrade - name: Test => (${{ matrix.test }} run: | - python ./bin/test_example.py ${{secrets.BASETEN_API_KEY}} ${{matrix.test}} + python _internal/bin/test_example.py ${{secrets.BASETEN_API_KEY}} ${{matrix.test}} report_to_slack: runs-on: ubuntu-latest if: always() && github.ref == 'refs/heads/main' diff --git a/.github/workflows/truss_deploy.yml b/.github/workflows/truss_deploy.yml index 188871205..0556bb3b9 100644 --- a/.github/workflows/truss_deploy.yml +++ b/.github/workflows/truss_deploy.yml @@ -31,4 +31,4 @@ jobs: - name: Run tests env: BASETEN_API_KEY: ${{ secrets.BASETEN_API_KEY }} - run: python bin/test_truss_deploy.py + run: python _internal/bin/test_truss_deploy.py diff --git a/.github/workflows/warm-chains.yml b/.github/workflows/warm-chains.yml index 4333850ec..ebe09a62f 100644 --- a/.github/workflows/warm-chains.yml +++ b/.github/workflows/warm-chains.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python environment - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.11.4' - name: Install dependencies (if any) @@ -29,5 +29,5 @@ jobs: EOF - name: Warm up chains run: | - truss chains deploy chains-examples/docs/poems/poems.py - truss chains deploy chains-examples/docs/audio-transcription/whisper_chainlet.py + truss chains deploy infrastructure/chains-examples/docs/poems/poems.py + truss chains deploy infrastructure/chains-examples/docs/audio-transcription/whisper_chainlet.py diff --git a/.gitignore b/.gitignore index 3babb0120..bd7d47898 100644 --- a/.gitignore +++ b/.gitignore @@ -1,163 +1,12 @@ .DS_Store -**/.DS_Store - -# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] -*$py.class - -# C extensions *.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ *.egg-info/ -.installed.cfg *.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments .env .venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy .mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.pytest_cache/ +.ruff_cache/ +validation_report.json diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index b9fb3f3e8..000000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa6cd8b17..709e1f468 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,24 +13,21 @@ repos: - id: check-merge-conflict - id: check-symlinks - id: debug-statements - exclude: ^sana/ + exclude: ^image/sana/ - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. rev: v0.11.6 hooks: - # Run the linter. - id: ruff args: [--fix, --fix-only] - # Run the formatter. - id: ruff-format types_or: [ python, pyi ] - repo: local hooks: - id: validate-ci - name: validate-ci-local - entry: poetry run python ./bin/validate_ci.py + name: validate-ci + entry: python _internal/bin/test_all.py language: python - additional_dependencies: ['truss', 'pyyaml'] + additional_dependencies: [truss, pyyaml] pass_filenames: false diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 0fb6db853..000000000 --- a/.tool-versions +++ /dev/null @@ -1,2 +0,0 @@ -python 3.11.9 -poetry 2.1.3 diff --git a/07-high-performance-dynamic-batching/README.md b/07-high-performance-dynamic-batching/README.md deleted file mode 100644 index 88b45a4e8..000000000 --- a/07-high-performance-dynamic-batching/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Dynamic Batching in Truss - -This repository contains an implementation designed to enable dynamic batching for machine learning models within the Truss framework. The core of this implementation lies in the `model/model.py` file, which introduces a `MlBatcher` class extending `AsyncBatcher`. This class is responsible for collecting individual prediction requests and processing them in batches, thereby improving throughput and efficiency. - -## Key Features - -- **Dynamic Batching:** The `MlBatcher` class dynamically batches incoming prediction requests, allowing for more efficient use of resources and faster response times. -- **Asynchronous Processing:** Utilizes asynchronous programming to handle concurrent prediction requests without blocking, ensuring high throughput. -- **Easy Integration:** Designed to be deployed as a normal Truss, making integration into existing projects straightforward. - -## Deployment - -To deploy this as a normal Truss, ensure you have the Truss CLI installed and configured. Then, follow these steps: - -1. Clone this repository to your local machine. -2. Navigate to the repository directory and build the Truss using the command `truss push --publish`. -3. Once the build completes, deploy the Truss to your desired environment. - -## Configuration - -The `config.yaml` file contains configuration options for the model, including the Python version, required packages, and runtime settings such as `predict_concurrency`. Adjust these settings as needed to optimize performance for your specific use case. - -## Testing - -The `test.py` file provides an example of how to send concurrent requests to the deployed model for testing purposes. Modify the URL and data as needed to match your deployment. - -## Conclusion - -This implementation showcases how dynamic batching can be seamlessly integrated into the Truss framework, providing a scalable and efficient solution for handling machine learning inference at scale. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding/README.md deleted file mode 100644 index 49d402ba2..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-modernbert-base-embedding - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-modernbert-base-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Alibaba-NLP/gte-modernbert-base is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: Alibaba-NLP/gte-modernbert-base - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md deleted file mode 100644 index 00fd6923d..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-Qwen2-1.5B-instruct-embedding - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-Qwen2-1.5B-instruct-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Alibaba-NLP/gte-Qwen2-1.5B-instruct is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: Alibaba-NLP/gte-Qwen2-1.5B-instruct - revision: main - source: HF - max_num_tokens: 131072 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md deleted file mode 100644 index 9d5e46b80..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-Qwen2-7B-instruct-embedding - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-Qwen2-7B-instruct-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Alibaba-NLP/gte-Qwen2-7B-instruct is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: Alibaba-NLP/gte-Qwen2-7B-instruct - revision: main - source: HF - max_num_tokens: 131072 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base/README.md deleted file mode 100644 index f4b4585e4..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-reranker-modernbert-base - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with Alibaba-NLP/gte-reranker-modernbert-base. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-reranker-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Reranker models may have at most one label, which contains the score of the reranking. - -Alibaba-NLP/gte-reranker-modernbert-base is a reranker model, used to re-rank a list of items, given a query. \nIt is frequently used in search engines, recommendation systems, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank`: -```json -{ - "query": "What is Baseten?", - "raw_scores": true, - "return_text": false, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -response = client.rerank( - query="What is Baseten?", - texts=["Deep Learning is ...", "Baseten is a fast inference provider"], - raw_scores=True, - return_text=False, - truncate=True, -) -print(response.data) -``` - -Sometimes, you may want to apply a custom template to the texts before reranking them and call the predict endpoint instead: - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - # Custom template function to apply to the texts - # a popular template might be "{query}\n{document}" - # or also chat-style templates like "User: {query}\nDocument: {document}" - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["What is baseten? A: Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - - -### Requests python library - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank", - json={ - "query": "What is Baseten?", - "raw_scores": True, - "return_text": False, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": True, - "truncation_direction": "Right" -} -``` -Returns: -```json -[ - { - "index": 0, - "score": 1, - "text": "Deep Learning is ..." - } -] -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/rerank` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI.com does not have a rerank endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - query: What is Baseten? - raw_scores: true - return_text: true - texts: - - Deep Learning is ... - - Baseten is a fast inference provider - truncate: true - truncation_direction: Right -model_name: BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: Alibaba-NLP/gte-reranker-modernbert-base - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /rerank - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large/README.md deleted file mode 100644 index 8fe64d72a..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with BAAI/bge-reranker-large - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with BAAI/bge-reranker-large. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Reranker models may have at most one label, which contains the score of the reranking. - -BAAI/bge-reranker-large is a reranker model, used to re-rank a list of items, given a query. \nIt is frequently used in search engines, recommendation systems, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-baai-bge-reranker-large-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank`: -```json -{ - "query": "What is Baseten?", - "raw_scores": true, - "return_text": false, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -response = client.rerank( - query="What is Baseten?", - texts=["Deep Learning is ...", "Baseten is a fast inference provider"], - raw_scores=True, - return_text=False, - truncate=True, -) -print(response.data) -``` - -Sometimes, you may want to apply a custom template to the texts before reranking them and call the predict endpoint instead: - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - # Custom template function to apply to the texts - # a popular template might be "{query}\n{document}" - # or also chat-style templates like "User: {query}\nDocument: {document}" - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["What is baseten? A: Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - - -### Requests python library - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank", - json={ - "query": "What is Baseten?", - "raw_scores": True, - "return_text": False, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": True, - "truncation_direction": "Right" -} -``` -Returns: -```json -[ - { - "index": 0, - "score": 1, - "text": "Deep Learning is ..." - } -] -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/rerank` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI.com does not have a rerank endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - query: What is Baseten? - raw_scores: true - return_text: true - texts: - - Deep Learning is ... - - Baseten is a fast inference provider - truncate: true - truncation_direction: Right -model_name: BEI-Bert-baai-bge-reranker-large-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: BAAI/bge-reranker-large - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /rerank - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m/README.md deleted file mode 100644 index 349a6af91..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with google/embeddinggemma-300m - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with google/embeddinggemma-300m. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -google/embeddinggemma-300m is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-google-embeddinggemma-300m-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-google-embeddinggemma-300m-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: google/embeddinggemma-300m - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct/README.md deleted file mode 100644 index 1a067245a..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with intfloat/multilingual-e5-large-instruct - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with intfloat/multilingual-e5-large-instruct. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [intfloat/multilingual-e5-large-instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -intfloat/multilingual-e5-large-instruct is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-intfloat-multilingual-e5-large-instruct-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-intfloat-multilingual-e5-large-instruct-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: intfloat/multilingual-e5-large-instruct - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en/README.md deleted file mode 100644 index 91770d5cc..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with jina-ai/jina-embeddings-v2-base-en - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with jina-ai/jina-embeddings-v2-base-en. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [jinaai/jina-embeddings-v2-base-en](https://huggingface.co/jinaai/jina-embeddings-v2-base-en). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -jinaai/jina-embeddings-v2-base-en is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-jina-ai-jina-embeddings-v2-base-en-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-jina-ai-jina-embeddings-v2-base-en-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: jinaai/jina-embeddings-v2-base-en - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code/README.md deleted file mode 100644 index adb9d9228..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with jinaai/jina-embeddings-v2-base-code - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with jinaai/jina-embeddings-v2-base-code. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [jinaai/jina-embeddings-v2-base-code](https://huggingface.co/jinaai/jina-embeddings-v2-base-code). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -jinaai/jina-embeddings-v2-base-code is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-jinaai-jina-embeddings-v2-base-code-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-jinaai-jina-embeddings-v2-base-code-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: jinaai/jina-embeddings-v2-base-code - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md deleted file mode 100644 index 7671ee6a2..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with mixedbread-ai/mxbai-embed-large-v1-embedding - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with mixedbread-ai/mxbai-embed-large-v1-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -mixedbread-ai/mxbai-embed-large-v1 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: mixedbread-ai/mxbai-embed-large-v1 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased/README.md deleted file mode 100644 index 748318cc2..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased/README.md +++ /dev/null @@ -1,187 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with NER/bert-base-ner-uncased - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with NER/bert-base-ner-uncased. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [baseten-admin/bert-base-ner-uncased](https://huggingface.co/baseten-admin/bert-base-ner-uncased). -Suitable models can be identified by the `ForTokenClassification` suffix in the model name. NER models classify each token in the input text into entity categories (e.g., PER, ORG, LOC) or 'O' (outside any entity). - -baseten-admin/bert-base-ner-uncased is a Named Entity Recognition (NER) model, used to identify and classify named entities in text. \nIt is frequently used for information extraction, entity linking, and document analysis. Common entities include persons, organizations, locations, dates, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-ner-bert-base-ner-uncased-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict_tokens` -```json -{ - "inputs": ["Apple is looking at buying U.K. startup for $1 billion"], - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) - -response = client.batch_post( - route="/predict_tokens", - payloads=[{ - "inputs": [["Apple is looking at buying U.K. startup for $1 billion"]], - "raw_scores": False, - "truncate": True, - "truncation_direction": "Right" - }] -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -response = requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict_tokens", - json={ - "inputs": [["Apple is looking at buying U.K. startup for $1 billion"]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -print(response.json()) -``` -Returns: -```json -[ - [ - { - "token": "[CLS]", - "token_id": 101, - "start": 0, - "end": 0, - "results": { - "O": 9.4140625, - "B-MISC": -1.15625, - "I-MISC": -0.859375, - "B-PER": -1.2744141, - "I-PER": -1.6552734, - "B-ORG": -0.88378906, - "I-ORG": -0.9345703, - "B-LOC": -1.2275391, - "I-LOC": -1.4042969 - } - }, - { - "token": "Apple", - "token_id": 6207, - "start": 0, - "end": 5, - "results": { - "B-ORG": 6.7578125, - "O": -1.7929688, - "B-LOC": 0.6015625, - "B-MISC": 0.2467041, - "B-PER": 0.17675781, - "I-ORG": -0.6484375, - "I-MISC": -1.9873047, - "I-LOC": -1.3808594, - "I-PER": -2.21875 - } - } - ] -] -``` -Important, this uses the `predict_tokens` endpoint for token-level classification. The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict_tokens` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a NER endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Apple is looking at buying U.K. startup for $1 billion - - - John works at Google in Mountain View, California - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-Bert-ner-bert-base-ner-uncased-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: baseten-admin/bert-base-ner-uncased - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /rerank - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5/README.md deleted file mode 100644 index 51ccd44c8..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with nomic-ai/nomic-embed-text-v1.5 - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with nomic-ai/nomic-embed-text-v1.5. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -nomic-ai/nomic-embed-text-v1.5 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-nomic-ai-nomic-embed-text-v1.5-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-nomic-ai-nomic-embed-text-v1.5-truss-example -python_version: py39 -resources: - accelerator: A10G - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: nomic-ai/nomic-embed-text-v1.5 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe/README.md deleted file mode 100644 index 17f648545..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with nomic-ai/nomic-embed-text-v2-moe - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with nomic-ai/nomic-embed-text-v2-moe. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nomic-ai/nomic-embed-text-v2-moe](https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -nomic-ai/nomic-embed-text-v2-moe is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-nomic-ai-nomic-embed-text-v2-moe-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-nomic-ai-nomic-embed-text-v2-moe-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: nomic-ai/nomic-embed-text-v2-moe - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b/README.md deleted file mode 100644 index 751e5c09c..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with nvidia/llama-embed-nemotron-8b - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with nvidia/llama-embed-nemotron-8b. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nvidia/llama-embed-nemotron-8b](https://huggingface.co/nvidia/llama-embed-nemotron-8b). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -nvidia/llama-embed-nemotron-8b is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-nvidia-llama-embed-nemotron-8b-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-nvidia-llama-embed-nemotron-8b-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: nvidia/llama-embed-nemotron-8b - revision: main - source: HF - max_num_tokens: 131072 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2/README.md deleted file mode 100644 index de54a8b4d..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with nvidia/llama-nemotron-embed-1b-v2 - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with nvidia/llama-nemotron-embed-1b-v2. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nvidia/llama-nemotron-embed-1b-v2](https://huggingface.co/nvidia/llama-nemotron-embed-1b-v2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -nvidia/llama-nemotron-embed-1b-v2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-nvidia-llama-nemotron-embed-1b-v2-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-nvidia-llama-nemotron-embed-1b-v2-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: nvidia/llama-nemotron-embed-1b-v2 - revision: main - source: HF - max_num_tokens: 131072 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2/README.md deleted file mode 100644 index fcea68db2..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with redis/langcache-embed-v2 - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with redis/langcache-embed-v2. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [redis/langcache-embed-v2](https://huggingface.co/redis/langcache-embed-v2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -redis/langcache-embed-v2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-redis-langcache-embed-v2-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-redis-langcache-embed-v2-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: redis/langcache-embed-v2 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding/README.md deleted file mode 100644 index b2b9f1f5f..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with sentence-transformers/all-MiniLM-L6-v2-embedding - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with sentence-transformers/all-MiniLM-L6-v2-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -sentence-transformers/all-MiniLM-L6-v2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: sentence-transformers/all-MiniLM-L6-v2 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1/README.md deleted file mode 100644 index 45bbd4308..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1/README.md +++ /dev/null @@ -1,187 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with tanaos/tanaos-NER-v1 - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with tanaos/tanaos-NER-v1. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [tanaos/tanaos-NER-v1](https://huggingface.co/tanaos/tanaos-NER-v1). -Suitable models can be identified by the `ForTokenClassification` suffix in the model name. NER models classify each token in the input text into entity categories (e.g., PER, ORG, LOC) or 'O' (outside any entity). - -tanaos/tanaos-NER-v1 is a Named Entity Recognition (NER) model, used to identify and classify named entities in text. \nIt is frequently used for information extraction, entity linking, and document analysis. Common entities include persons, organizations, locations, dates, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-tanaos-tanaos-ner-v1-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict_tokens` -```json -{ - "inputs": ["Apple is looking at buying U.K. startup for $1 billion"], - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) - -response = client.batch_post( - route="/predict_tokens", - payloads=[{ - "inputs": [["Apple is looking at buying U.K. startup for $1 billion"]], - "raw_scores": False, - "truncate": True, - "truncation_direction": "Right" - }] -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -response = requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict_tokens", - json={ - "inputs": [["Apple is looking at buying U.K. startup for $1 billion"]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -print(response.json()) -``` -Returns: -```json -[ - [ - { - "token": "[CLS]", - "token_id": 101, - "start": 0, - "end": 0, - "results": { - "O": 9.4140625, - "B-MISC": -1.15625, - "I-MISC": -0.859375, - "B-PER": -1.2744141, - "I-PER": -1.6552734, - "B-ORG": -0.88378906, - "I-ORG": -0.9345703, - "B-LOC": -1.2275391, - "I-LOC": -1.4042969 - } - }, - { - "token": "Apple", - "token_id": 6207, - "start": 0, - "end": 5, - "results": { - "B-ORG": 6.7578125, - "O": -1.7929688, - "B-LOC": 0.6015625, - "B-MISC": 0.2467041, - "B-PER": 0.17675781, - "I-ORG": -0.6484375, - "I-MISC": -1.9873047, - "I-LOC": -1.3808594, - "I-PER": -2.21875 - } - } - ] -] -``` -Important, this uses the `predict_tokens` endpoint for token-level classification. The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict_tokens` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a NER endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Apple is looking at buying U.K. startup for $1 billion - - - John works at Google in Mountain View, California - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-Bert-tanaos-tanaos-ner-v1-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: tanaos/tanaos-NER-v1 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /rerank - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2/README.md deleted file mode 100644 index 3233c22d5..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with TaylorAI/bge-micro-v2 - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with TaylorAI/bge-micro-v2. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [TaylorAI/bge-micro-v2](https://huggingface.co/TaylorAI/bge-micro-v2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -TaylorAI/bge-micro-v2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-taylorai-bge-micro-v2-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-taylorai-bge-micro-v2-truss-example -python_version: py39 -resources: - accelerator: A10G - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: TaylorAI/bge-micro-v2 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano/README.md deleted file mode 100644 index 954dc8fd7..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI-Bert (Baseten-Embeddings-Inference-BERT) with voyageai/voyage-4-nano - -This is a Deployment for BEI-Bert (Baseten-Embeddings-Inference-BERT) with voyageai/voyage-4-nano. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [voyageai/voyage-4-nano](https://huggingface.co/voyageai/voyage-4-nano). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -voyageai/voyage-4-nano is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-Bert-voyageai-voyage-4-nano-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-Bert-voyageai-voyage-4-nano-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder_bert - checkpoint_repository: - repo: voyageai/voyage-4-nano - revision: main - source: HF - max_num_tokens: 40960 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md deleted file mode 100644 index 18f7e2650..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with allenai/Llama-3.1-Tulu-3-8B-Reward-Model - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with allenai/Llama-3.1-Tulu-3-8B-Reward-Model. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [allenai/Llama-3.1-Tulu-3-8B-RM](https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B-RM). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -allenai/Llama-3.1-Tulu-3-8B-RM is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: allenai/Llama-3.1-Tulu-3-8B-RM - revision: main - source: HF - max_num_tokens: 131072 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8/README.md deleted file mode 100644 index 7c5af8417..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with BAAI/bge-en-icl-embedding - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with BAAI/bge-en-icl-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-en-icl](https://huggingface.co/BAAI/bge-en-icl). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -BAAI/bge-en-icl is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-baai-bge-en-icl-embedding-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-baai-bge-en-icl-embedding-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: BAAI/bge-en-icl - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 2 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding/README.md deleted file mode 100644 index 9787f7583..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with BAAI/bge-large-en-v1.5-embedding - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with BAAI/bge-large-en-v1.5-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-large-en-v1.5](https://huggingface.co/BAAI/bge-large-en-v1.5). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -BAAI/bge-large-en-v1.5 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-baai-bge-large-en-v1.5-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-baai-bge-large-en-v1.5-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: BAAI/bge-large-en-v1.5 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense/README.md deleted file mode 100644 index fde3603e4..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with BAAI/bge-m3-embedding-dense - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with BAAI/bge-m3-embedding-dense. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -BAAI/bge-m3 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-baai-bge-m3-embedding-dense-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-baai-bge-m3-embedding-dense-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: BAAI/bge-m3 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding/README.md deleted file mode 100644 index 5c479a4c4..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with BAAI/bge-multilingual-gemma2-multilingual-embedding - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with BAAI/bge-multilingual-gemma2-multilingual-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-multilingual-gemma2](https://huggingface.co/BAAI/bge-multilingual-gemma2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -BAAI/bge-multilingual-gemma2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-baai-bge-multilingual-gemma2-multilingual-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-baai-bge-multilingual-gemma2-multilingual-embedding-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: BAAI/bge-multilingual-gemma2 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large/README.md deleted file mode 100644 index 3525c7256..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with BAAI/bge-reranker-large - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with BAAI/bge-reranker-large. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Reranker models may have at most one label, which contains the score of the reranking. - -BAAI/bge-reranker-large is a reranker model, used to re-rank a list of items, given a query. \nIt is frequently used in search engines, recommendation systems, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-baai-bge-reranker-large-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank`: -```json -{ - "query": "What is Baseten?", - "raw_scores": true, - "return_text": false, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -response = client.rerank( - query="What is Baseten?", - texts=["Deep Learning is ...", "Baseten is a fast inference provider"], - raw_scores=True, - return_text=False, - truncate=True, -) -print(response.data) -``` - -Sometimes, you may want to apply a custom template to the texts before reranking them and call the predict endpoint instead: - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - # Custom template function to apply to the texts - # a popular template might be "{query}\n{document}" - # or also chat-style templates like "User: {query}\nDocument: {document}" - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["What is baseten? A: Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - - -### Requests python library - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank", - json={ - "query": "What is Baseten?", - "raw_scores": True, - "return_text": False, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": True, - "truncation_direction": "Right" -} -``` -Returns: -```json -[ - { - "index": 0, - "score": 1, - "text": "Deep Learning is ..." - } -] -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/rerank` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI.com does not have a rerank endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - query: What is Baseten? - raw_scores: true - return_text: true - texts: - - Deep Learning is ... - - Baseten is a fast inference provider - truncate: true - truncation_direction: Right -model_name: BEI-baai-bge-reranker-large-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: BAAI/bge-reranker-large - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /rerank - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual/README.md deleted file mode 100644 index e437588f8..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with BAAI/bge-reranker-v2-m3-multilingual - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with BAAI/bge-reranker-v2-m3-multilingual. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Reranker models may have at most one label, which contains the score of the reranking. - -BAAI/bge-reranker-v2-m3 is a reranker model, used to re-rank a list of items, given a query. \nIt is frequently used in search engines, recommendation systems, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-baai-bge-reranker-v2-m3-multilingual-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank`: -```json -{ - "query": "What is Baseten?", - "raw_scores": true, - "return_text": false, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -response = client.rerank( - query="What is Baseten?", - texts=["Deep Learning is ...", "Baseten is a fast inference provider"], - raw_scores=True, - return_text=False, - truncate=True, -) -print(response.data) -``` - -Sometimes, you may want to apply a custom template to the texts before reranking them and call the predict endpoint instead: - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - # Custom template function to apply to the texts - # a popular template might be "{query}\n{document}" - # or also chat-style templates like "User: {query}\nDocument: {document}" - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["What is baseten? A: Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - - -### Requests python library - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank", - json={ - "query": "What is Baseten?", - "raw_scores": True, - "return_text": False, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": True, - "truncation_direction": "Right" -} -``` -Returns: -```json -[ - { - "index": 0, - "score": 1, - "text": "Deep Learning is ..." - } -] -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/rerank` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI.com does not have a rerank endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - query: What is Baseten? - raw_scores: true - return_text: true - texts: - - Deep Learning is ... - - Baseten is a fast inference provider - truncate: true - truncation_direction: Right -model_name: BEI-baai-bge-reranker-v2-m3-multilingual-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: BAAI/bge-reranker-v2-m3 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /rerank - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md deleted file mode 100644 index d65b060d7..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification](https://huggingface.co/baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification - revision: main - source: HF - max_num_tokens: 16384 - num_builder_gpus: 2 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8/README.md deleted file mode 100644 index a9d3c3f19..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with codefuse-ai/F2LLM-4B-embedding - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with codefuse-ai/F2LLM-4B-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [codefuse-ai/F2LLM-4B](https://huggingface.co/codefuse-ai/F2LLM-4B). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -codefuse-ai/F2LLM-4B is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-codefuse-ai-f2llm-4b-embedding-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-codefuse-ai-f2llm-4b-embedding-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: codefuse-ai/F2LLM-4B - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 2 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md deleted file mode 100644 index 442d6e974..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with intfloat/e5-mistral-7b-instruct-embedding - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with intfloat/e5-mistral-7b-instruct-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [intfloat/e5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -intfloat/e5-mistral-7b-instruct is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: intfloat/e5-mistral-7b-instruct - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 2 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8/README.md deleted file mode 100644 index d89fde450..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with jinaai/jina-code-embeddings-0.5b - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with jinaai/jina-code-embeddings-0.5b. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [jinaai/jina-code-embeddings-0.5b](https://huggingface.co/jinaai/jina-code-embeddings-0.5b). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -jinaai/jina-code-embeddings-0.5b is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-jinaai-jina-code-embeddings-0.5b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-jinaai-jina-code-embeddings-0.5b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: jinaai/jina-code-embeddings-0.5b - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md deleted file mode 100644 index 77fa774fb..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with mixedbread-ai/mxbai-embed-large-v1-embedding - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with mixedbread-ai/mxbai-embed-large-v1-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -mixedbread-ai/mxbai-embed-large-v1 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-mixedbread-ai-mxbai-embed-large-v1-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-mixedbread-ai-mxbai-embed-large-v1-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: mixedbread-ai/mxbai-embed-large-v1 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md deleted file mode 100644 index a422be568..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with mixedbread-ai/mxbai-rerank-base-v2-reranker - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with mixedbread-ai/mxbai-rerank-base-v2-reranker. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/mxbai-rerank-base-v2-seq](https://huggingface.co/michaelfeil/mxbai-rerank-base-v2-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/mxbai-rerank-base-v2-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/mxbai-rerank-base-v2-seq - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md deleted file mode 100644 index 24988153f..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with mixedbread-ai/mxbai-rerank-large-v2-reranker - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with mixedbread-ai/mxbai-rerank-large-v2-reranker. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/mxbai-rerank-large-v2-seq](https://huggingface.co/michaelfeil/mxbai-rerank-large-v2-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/mxbai-rerank-large-v2-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/mxbai-rerank-large-v2-seq - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker/README.md deleted file mode 100644 index 19aea0756..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with ncbi/MedCPT-Cross-Encoder-reranker - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with ncbi/MedCPT-Cross-Encoder-reranker. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [ncbi/MedCPT-Cross-Encoder](https://huggingface.co/ncbi/MedCPT-Cross-Encoder). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Reranker models may have at most one label, which contains the score of the reranking. - -ncbi/MedCPT-Cross-Encoder is a reranker model, used to re-rank a list of items, given a query. \nIt is frequently used in search engines, recommendation systems, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-ncbi-medcpt-cross-encoder-reranker-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank`: -```json -{ - "query": "What is Baseten?", - "raw_scores": true, - "return_text": false, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -response = client.rerank( - query="What is Baseten?", - texts=["Deep Learning is ...", "Baseten is a fast inference provider"], - raw_scores=True, - return_text=False, - truncate=True, -) -print(response.data) -``` - -Sometimes, you may want to apply a custom template to the texts before reranking them and call the predict endpoint instead: - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - # Custom template function to apply to the texts - # a popular template might be "{query}\n{document}" - # or also chat-style templates like "User: {query}\nDocument: {document}" - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["What is baseten? A: Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - - -### Requests python library - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank", - json={ - "query": "What is Baseten?", - "raw_scores": True, - "return_text": False, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": True, - "truncation_direction": "Right" -} -``` -Returns: -```json -[ - { - "index": 0, - "score": 1, - "text": "Deep Learning is ..." - } -] -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/rerank` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI.com does not have a rerank endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - query: What is Baseten? - raw_scores: true - return_text: true - texts: - - Deep Learning is ... - - Baseten is a fast inference provider - truncate: true - truncation_direction: Right -model_name: BEI-ncbi-medcpt-cross-encoder-reranker-truss-example -python_version: py39 -resources: - accelerator: A10G - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: ncbi/MedCPT-Cross-Encoder - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /rerank - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8/README.md deleted file mode 100644 index 33d1983c9..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with nomic-ai/nomic-embed-code - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with nomic-ai/nomic-embed-code. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nomic-ai/nomic-embed-code](https://huggingface.co/nomic-ai/nomic-embed-code). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -nomic-ai/nomic-embed-code is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-nomic-ai-nomic-embed-code-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-nomic-ai-nomic-embed-code-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: nomic-ai/nomic-embed-code - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification/README.md deleted file mode 100644 index 3f0227fbd..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification/README.md +++ /dev/null @@ -1,166 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with papluca/xlm-roberta-base-language-detection-classification - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with papluca/xlm-roberta-base-language-detection-classification. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [papluca/xlm-roberta-base-language-detection](https://huggingface.co/papluca/xlm-roberta-base-language-detection). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -papluca/xlm-roberta-base-language-detection is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-papluca-xlm-roberta-base-language-detection-classification-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-papluca-xlm-roberta-base-language-detection-classification-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: papluca/xlm-roberta-base-language-detection - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md deleted file mode 100644 index 02e60bd4f..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-0.6B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-0.6B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-0.6B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-0.6B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4/README.md deleted file mode 100644 index dbfef7320..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4/README.md +++ /dev/null @@ -1,180 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-4B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-4B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-4B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-4B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-4B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-4b-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-4b-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-4B-auto - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp4 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8/README.md deleted file mode 100644 index 913dbeedd..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-4B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-4B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-4B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-4B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-4B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-4b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-4b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-4B-auto - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8/README.md deleted file mode 100644 index c0d510c2f..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-8B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-8B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-8B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-8B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-8B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-8b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-8b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-8B-auto - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md deleted file mode 100644 index b5797c630..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-0.6B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-0.6B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Reranker-0.6B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-0.6B-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/Qwen3-Reranker-0.6B-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-reranker-0.6b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-qwen-qwen3-reranker-0.6b-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-0.6B-seq - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8/README.md deleted file mode 100644 index 459cb3ce1..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-4B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-4B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Reranker-4B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-4B-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/Qwen3-Reranker-4B-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-reranker-4b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-qwen-qwen3-reranker-4b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-4B-seq - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4/README.md deleted file mode 100644 index e6ad83e09..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-8B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-8B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Reranker-8B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-8B-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/Qwen3-Reranker-8B-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-reranker-8b-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-qwen-qwen3-reranker-8b-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-8B-seq - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp4 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8/README.md deleted file mode 100644 index a1d7d7d28..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-8B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-8B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Reranker-8B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-8B-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/Qwen3-Reranker-8B-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-reranker-8b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-qwen-qwen3-reranker-8b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-8B-seq - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8/README.md deleted file mode 100644 index ef8e1b817..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Salesforce/SFR-Embedding-Mistral - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Salesforce/SFR-Embedding-Mistral. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Salesforce/SFR-Embedding-Mistral](https://huggingface.co/Salesforce/SFR-Embedding-Mistral). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Salesforce/SFR-Embedding-Mistral is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-salesforce-sfr-embedding-mistral-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-salesforce-sfr-embedding-mistral-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: Salesforce/SFR-Embedding-Mistral - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification/README.md deleted file mode 100644 index d5bf004d5..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification/README.md +++ /dev/null @@ -1,166 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with SamLowe/roberta-base-go_emotions-classification - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with SamLowe/roberta-base-go_emotions-classification. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [SamLowe/roberta-base-go_emotions](https://huggingface.co/SamLowe/roberta-base-go_emotions). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -SamLowe/roberta-base-go_emotions is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-samlowe-roberta-base-go_emotions-classification-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-samlowe-roberta-base-go_emotions-classification-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: SamLowe/roberta-base-go_emotions - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md deleted file mode 100644 index 398996972..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Skywork/Skywork-Reward-Llama-3.1-8B-v0.2-Reward-Model - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Skywork/Skywork-Reward-Llama-3.1-8B-v0.2-Reward-Model. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Skywork/Skywork-Reward-Llama-3.1-8B-v0.2](https://huggingface.co/Skywork/Skywork-Reward-Llama-3.1-8B-v0.2). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - -### Requests python library -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - - Baseten is a fast inference provider - - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 - revision: main - source: HF - max_num_tokens: 131072 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0/README.md deleted file mode 100644 index e85501e4f..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Snowflake/snowflake-arctic-embed-l-v2.0 - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Snowflake/snowflake-arctic-embed-l-v2.0. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Snowflake/snowflake-arctic-embed-l-v2.0](https://huggingface.co/Snowflake/snowflake-arctic-embed-l-v2.0). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Snowflake/snowflake-arctic-embed-l-v2.0 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-snowflake-snowflake-arctic-embed-l-v2.0-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-snowflake-snowflake-arctic-embed-l-v2.0-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: Snowflake/snowflake-arctic-embed-l-v2.0 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding/README.md deleted file mode 100644 index f30fa8fdc..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with WhereIsAI/UAE-Large-V1-embedding - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with WhereIsAI/UAE-Large-V1-embedding. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [WhereIsAI/UAE-Large-V1](https://huggingface.co/WhereIsAI/UAE-Large-V1). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -WhereIsAI/UAE-Large-V1 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-whereisai-uae-large-v1-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-whereisai-uae-large-v1-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: WhereIsAI/UAE-Large-V1 - revision: main - source: HF - max_num_tokens: 16384 - runtime: - webserver_default_route: /v1/embeddings - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md deleted file mode 100644 index 0f2df2f5d..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Baseten Inference Stack with deepseek-ai/DeepSeek-R1-Distill-Llama-70B - -This is a Deployment for Baseten Inference Stack with deepseek-ai/DeepSeek-R1-Distill-Llama-70B. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -deepseek-ai/DeepSeek-R1-Distill-Llama-70B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: deepseek-ai/DeepSeek-R1-Distill-Llama-70B - revision: main - source: HF - quantization_type: fp4_kv - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md deleted file mode 100644 index 388de80ff..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Baseten Inference Stack with meta-llama/Llama-3.2-3B-Instruct - -This is a Deployment for Baseten Inference Stack with meta-llama/Llama-3.2-3B-Instruct. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.2-3B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4_mlp_only`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: meta-llama/Llama-3.2-3B-Instruct - revision: main - source: HF - quantization_type: fp4_mlp_only - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8/README.md deleted file mode 100644 index 2a5883753..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# Baseten Inference Stack with meta-llama/Llama-3.2-3B-Instruct - -This is a Deployment for Baseten Inference Stack with meta-llama/Llama-3.2-3B-Instruct. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.2-3B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-meta-llama-llama-3.2-3b-instruct-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-meta-llama-llama-3.2-3b-instruct-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: meta-llama/Llama-3.2-3B-Instruct - revision: main - source: HF - quantization_type: fp8_kv - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4/README.md deleted file mode 100644 index a4a9f741b..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Baseten Inference Stack with meta-llama/Llama-3.3-70B-Instruct - -This is a Deployment for Baseten Inference Stack with meta-llama/Llama-3.3-70B-Instruct. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.3-70B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-meta-llama-llama-3.3-70b-instruct-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-meta-llama-llama-3.3-70b-instruct-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: meta-llama/Llama-3.3-70B-Instruct - revision: main - source: HF - quantization_type: fp4 - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4/README.md deleted file mode 100644 index 0f07efb6d..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Baseten Inference Stack with nvidia/Llama-3.1-8B-Instruct-FP4 - -This is a Deployment for Baseten Inference Stack with nvidia/Llama-3.1-8B-Instruct-FP4. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nvidia/Llama-3.1-8B-Instruct-FP4](https://huggingface.co/nvidia/Llama-3.1-8B-Instruct-FP4). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -nvidia/Llama-3.1-8B-Instruct-FP4 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-nvidia-llama-3.1-8b-instruct-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-nvidia-llama-3.1-8b-instruct-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: nvidia/Llama-3.1-8B-Instruct-FP4 - revision: main - source: HF - quantization_type: no_quant - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4/README.md deleted file mode 100644 index aba79f029..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Baseten Inference Stack with nvidia/Qwen3-30B-A3B-FP4 - -This is a Deployment for Baseten Inference Stack with nvidia/Qwen3-30B-A3B-FP4. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nvidia/Qwen3-30B-A3B-FP4](https://huggingface.co/nvidia/Qwen3-30B-A3B-FP4). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -nvidia/Qwen3-30B-A3B-FP4 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-nvidia-qwen3-30b-a3b-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-nvidia-qwen3-30b-a3b-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: nvidia/Qwen3-30B-A3B-FP4 - revision: main - source: HF - quantization_type: no_quant - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4/README.md deleted file mode 100644 index 08d12366d..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Baseten Inference Stack with nvidia/Qwen3-8B-FP4 - -This is a Deployment for Baseten Inference Stack with nvidia/Qwen3-8B-FP4. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nvidia/Qwen3-8B-FP4](https://huggingface.co/nvidia/Qwen3-8B-FP4). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -nvidia/Qwen3-8B-FP4 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-nvidia-qwen3-8b-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-nvidia-qwen3-8b-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: nvidia/Qwen3-8B-FP4 - revision: main - source: HF - quantization_type: no_quant - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4/README.md deleted file mode 100644 index 4b4cae484..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# Baseten Inference Stack with Qwen/Qwen2.5-Coder-7B-Instruct - -This is a Deployment for Baseten Inference Stack with Qwen/Qwen2.5-Coder-7B-Instruct. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen2.5-Coder-7B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-qwen-qwen2.5-coder-7b-instruct-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-qwen-qwen2.5-coder-7b-instruct-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: Qwen/Qwen2.5-Coder-7B-Instruct - revision: main - source: HF - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp4 - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct/README.md deleted file mode 100644 index 90f588260..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# Baseten Inference Stack with Qwen/Qwen2.5-Coder-7B-Instruct - -This is a Deployment for Baseten Inference Stack with Qwen/Qwen2.5-Coder-7B-Instruct. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen2.5-Coder-7B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-qwen-qwen2.5-coder-7b-instruct-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-qwen-qwen2.5-coder-7b-instruct-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: Qwen/Qwen2.5-Coder-7B-Instruct - revision: main - source: HF - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: no_quant - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4/README.md deleted file mode 100644 index f7ee35ec1..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Baseten Inference Stack with Qwen/Qwen3-32B - -This is a Deployment for Baseten Inference Stack with Qwen/Qwen3-32B. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-32B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-qwen-qwen3-32b-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-qwen-qwen3-32b-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: Qwen/Qwen3-32B - revision: main - source: HF - quantization_type: fp4_kv - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8/README.md deleted file mode 100644 index ce82ab191..000000000 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# Baseten Inference Stack with Qwen/Qwen3-4B - -This is a Deployment for Baseten Inference Stack with Qwen/Qwen3-4B. Baseten Inference Stack is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Baseten Inference Stack you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs -- *fp4 quantization* deployments on B200 GPUs to get even more speed - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-4B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BISV2-qwen-qwen3-4b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: BISV2-qwen-qwen3-4b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - checkpoint_repository: - repo: Qwen/Qwen3-4B - revision: main - source: HF - quantization_type: fp8_kv - runtime: - max_batch_size: 32 - max_num_tokens: 32768 - max_seq_len: 32768 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md deleted file mode 100644 index ad96330af..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# TensorRT-LLM Briton with deepseek-ai/DeepSeek-R1-Distill-Llama-70B - -This is a Deployment for TensorRT-LLM Briton with deepseek-ai/DeepSeek-R1-Distill-Llama-70B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -deepseek-ai/DeepSeek-R1-Distill-Llama-70B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100:2 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: deepseek-ai/DeepSeek-R1-Distill-Llama-70B - revision: main - source: HF - max_seq_len: 131072 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 2 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md deleted file mode 100644 index 67bf8b5a9..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# TensorRT-LLM Briton with deepseek-ai/DeepSeek-R1-Distill-Qwen-32B - -This is a Deployment for TensorRT-LLM Briton with deepseek-ai/DeepSeek-R1-Distill-Qwen-32B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -deepseek-ai/DeepSeek-R1-Distill-Qwen-32B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: deepseek-ai/DeepSeek-R1-Distill-Qwen-32B - revision: main - source: HF - max_seq_len: 131072 - num_builder_gpus: 4 - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp8 - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it/README.md deleted file mode 100644 index 80b69da8a..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# TensorRT-LLM Briton with google/gemma-3-1b-it - -This is a Deployment for TensorRT-LLM Briton with google/gemma-3-1b-it. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [unsloth/gemma-3-1b-it](https://huggingface.co/unsloth/gemma-3-1b-it). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -unsloth/gemma-3-1b-it is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-google-gemma-3-1b-it-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-google-gemma-3-1b-it-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: unsloth/gemma-3-1b-it - revision: main - source: HF - max_seq_len: 32768 - quantization_type: no_quant - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it/README.md deleted file mode 100644 index dccff493b..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# TensorRT-LLM Briton with google/gemma-3-270m-it - -This is a Deployment for TensorRT-LLM Briton with google/gemma-3-270m-it. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -google/gemma-3-270m-it is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-google-gemma-3-270m-it-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-google-gemma-3-270m-it-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: google/gemma-3-270m-it - revision: main - source: HF - max_seq_len: 32768 - quantization_type: no_quant - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead/README.md deleted file mode 100644 index 1b14d9bfe..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# TensorRT-LLM Briton with google/gemma-3-27b-it-speculative-lookahead - -This is a Deployment for TensorRT-LLM Briton with google/gemma-3-27b-it-speculative-lookahead. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [baseten/gemma-3-27b-causallm-it](https://huggingface.co/baseten/gemma-3-27b-causallm-it). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -baseten/gemma-3-27b-causallm-it is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-google-gemma-3-27b-it-speculative-lookahead-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-google-gemma-3-27b-it-speculative-lookahead-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: baseten/gemma-3-27b-causallm-it - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 131072 - max_seq_len: 131072 - quantization_type: no_quant - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 8 - lookahead_verification_set_size: 3 - lookahead_windows_size: 3 - num_draft_tokens: 41 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it/README.md deleted file mode 100644 index f0f5f39c9..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# TensorRT-LLM Briton with google/gemma-3-27b-it - -This is a Deployment for TensorRT-LLM Briton with google/gemma-3-27b-it. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [baseten/gemma-3-27b-causallm-it](https://huggingface.co/baseten/gemma-3-27b-causallm-it). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -baseten/gemma-3-27b-causallm-it is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-google-gemma-3-27b-it-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-google-gemma-3-27b-it-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: baseten/gemma-3-27b-causallm-it - revision: main - source: HF - max_seq_len: 131072 - quantization_type: no_quant - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8/README.md deleted file mode 100644 index 44d49ec99..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.1-405B - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.1-405B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.1-405B](https://huggingface.co/meta-llama/Llama-3.1-405B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.1-405B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.1-405b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.1-405b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100:8 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.1-405B - revision: main - source: HF - max_seq_len: 131072 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 8 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md deleted file mode 100644 index a3b93f923..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.1-8B-Instruct-with-speculative-lookahead-decoding - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.1-8B-Instruct-with-speculative-lookahead-decoding. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.1-8B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.1-8B-Instruct - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 131072 - max_seq_len: 131072 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 8 - lookahead_verification_set_size: 3 - lookahead_windows_size: 3 - num_draft_tokens: 41 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8/README.md deleted file mode 100644 index f15bc82be..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.2-1B-Instruct - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.2-1B-Instruct. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.2-1B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.2-1b-instruct-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.2-1b-instruct-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.2-1B-Instruct - revision: main - source: HF - max_seq_len: 131072 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md deleted file mode 100644 index 36ff57234..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md +++ /dev/null @@ -1,174 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.2-3B-Instruct-calib-dataset - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.2-3B-Instruct-calib-dataset. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.2-3B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.2-3B-Instruct - revision: main - source: HF - max_seq_len: 131072 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_config: - calib_dataset: baseten/quant_calibration_dataset_v1 - quantization_type: fp8_kv - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8/README.md deleted file mode 100644 index e70711dce..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.2-3B-Instruct - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.2-3B-Instruct. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.2-3B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.2-3b-instruct-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.2-3b-instruct-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.2-3B-Instruct - revision: main - source: HF - max_seq_len: 131072 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct/README.md deleted file mode 100644 index 51edf8821..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.2-3B-Instruct - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.2-3B-Instruct. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.2-3B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.2-3b-instruct-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.2-3b-instruct-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.2-3B-Instruct - revision: main - source: HF - max_seq_len: 131072 - quantization_type: no_quant - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4/README.md deleted file mode 100644 index aee9a97c4..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.3-70B-Instruct - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.3-70B-Instruct. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.3-70B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.3-70b-instruct-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.3-70b-instruct-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.3-70B-Instruct - revision: main - source: HF - max_seq_len: 131072 - num_builder_gpus: 4 - quantization_type: fp4 - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8/README.md deleted file mode 100644 index 13d8b2b65..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.3-70B-Instruct - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.3-70B-Instruct. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.3-70B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.3-70b-instruct-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.3-70b-instruct-fp8-truss-example -python_version: py39 -resources: - accelerator: H100:2 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.3-70B-Instruct - revision: main - source: HF - max_seq_len: 131072 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 2 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md deleted file mode 100644 index c401138ee..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.3-70B-Instruct-tp4 - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.3-70B-Instruct-tp4. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.3-70B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8-truss-example -python_version: py39 -resources: - accelerator: H100:4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.3-70B-Instruct - revision: main - source: HF - max_seq_len: 131072 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 4 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8/README.md deleted file mode 100644 index f35cf47b1..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# TensorRT-LLM Briton with microsoft/phi-4 - -This is a Deployment for TensorRT-LLM Briton with microsoft/phi-4. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [unsloth/phi-4](https://huggingface.co/unsloth/phi-4). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -unsloth/phi-4 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-microsoft-phi-4-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-microsoft-phi-4-fp8-truss-example -python_version: py39 -resources: - accelerator: L4:2 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: unsloth/phi-4 - revision: main - source: HF - max_seq_len: 16384 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 2 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3/README.md deleted file mode 100644 index 282dd9fec..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# TensorRT-LLM Briton with mistralai/Mistral-7B-Instruct-v0.3 - -This is a Deployment for TensorRT-LLM Briton with mistralai/Mistral-7B-Instruct-v0.3. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -mistralai/Mistral-7B-Instruct-v0.3 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-mistralai-mistral-7b-instruct-v0.3-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-mistralai-mistral-7b-instruct-v0.3-truss-example -python_version: py39 -resources: - accelerator: A10G:2 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: mistralai/Mistral-7B-Instruct-v0.3 - revision: main - source: HF - max_seq_len: 32768 - quantization_type: no_quant - tensor_parallel_count: 2 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8/README.md deleted file mode 100644 index b6e3eff92..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# TensorRT-LLM Briton with mistralai/Mistral-Small-24B-Instruct-2501 - -This is a Deployment for TensorRT-LLM Briton with mistralai/Mistral-Small-24B-Instruct-2501. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -mistralai/Mistral-Small-24B-Instruct-2501 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-mistralai-mistral-small-24b-instruct-2501-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-mistralai-mistral-small-24b-instruct-2501-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: mistralai/Mistral-Small-24B-Instruct-2501 - revision: main - source: HF - max_seq_len: 32768 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8/README.md deleted file mode 100644 index f11d57006..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen2.5-72B-Instruct-tp2 - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen2.5-72B-Instruct-tp2. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen2.5-72B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen2.5-72b-instruct-tp2-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen2.5-72b-instruct-tp2-fp8-truss-example -python_version: py39 -resources: - accelerator: H100:2 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen2.5-72B-Instruct - revision: main - source: HF - max_seq_len: 32768 - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp8 - tensor_parallel_count: 2 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md deleted file mode 100644 index c11eafc02..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md +++ /dev/null @@ -1,182 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen2.5-7B-Instruct-with-speculative-lookahead-decoding - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen2.5-7B-Instruct-with-speculative-lookahead-decoding. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen2.5-7B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen2.5-7B-Instruct - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 32768 - num_builder_gpus: 4 - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp8 - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 8 - lookahead_verification_set_size: 3 - lookahead_windows_size: 3 - num_draft_tokens: 41 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md deleted file mode 100644 index a4e7260dc..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen2.5-Coder-7B-Instruct-calib-dataset - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen2.5-Coder-7B-Instruct-calib-dataset. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen2.5-Coder-7B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4_mlp_only`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen2.5-Coder-7B-Instruct - revision: main - source: HF - max_seq_len: 32768 - num_builder_gpus: 4 - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp4_mlp_only - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md deleted file mode 100644 index 850c58dc1..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md +++ /dev/null @@ -1,182 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen2.5-Coder-7B-Instruct-min-latency - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen2.5-Coder-7B-Instruct-min-latency. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen2.5-Coder-7B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen2.5-Coder-7B-Instruct - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 32768 - num_builder_gpus: 4 - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp8 - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 32 - lookahead_verification_set_size: 1 - lookahead_windows_size: 1 - num_draft_tokens: 61 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md deleted file mode 100644 index 2fa13067b..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-235B-A22B-Instruct-2507 - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-235B-A22B-Instruct-2507. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-235B-A22B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-235B-A22B-Instruct-2507 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8-truss-example -python_version: py39 -resources: - accelerator: H100:8 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-235B-A22B-Instruct-2507 - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 262144 - max_seq_len: 262144 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 32 - lookahead_verification_set_size: 1 - lookahead_windows_size: 1 - num_draft_tokens: 61 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 8 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8/README.md deleted file mode 100644 index 723d21364..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-30B-A3B - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-30B-A3B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-30B-A3B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-30b-a3b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-30b-a3b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-30B-A3B - revision: main - source: HF - max_seq_len: 40960 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md deleted file mode 100644 index 885c5b9ea..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-30B-A3B-Instruct-2507 - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-30B-A3B-Instruct-2507. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-30B-A3B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-30B-A3B-Instruct-2507 is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-30B-A3B-Instruct-2507 - revision: main - source: HF - max_seq_len: 262144 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b/README.md deleted file mode 100644 index c3a55f19e..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-30B-A3B - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-30B-A3B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-30B-A3B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-30b-a3b-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-30b-a3b-truss-example -python_version: py39 -resources: - accelerator: H100:2 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-30B-A3B - revision: main - source: HF - max_seq_len: 40960 - quantization_type: no_quant - tensor_parallel_count: 2 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only/README.md deleted file mode 100644 index 0f5ae98a3..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-32B - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-32B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-32B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-32b-fp4-mlp-only-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4_mlp_only`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-32b-fp4-mlp-only-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-32B - revision: main - source: HF - max_seq_len: 40960 - num_builder_gpus: 4 - quantization_type: fp4_mlp_only - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4/README.md deleted file mode 100644 index a37b059cd..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-32B - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-32B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-32B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-32b-fp4-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp4_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-32b-fp4-truss-example -python_version: py39 -resources: - accelerator: B200 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-32B - revision: main - source: HF - max_seq_len: 40960 - num_builder_gpus: 4 - quantization_type: fp4_kv - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8/README.md deleted file mode 100644 index 265a4731b..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-32B - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-32B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-32B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-32b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-32b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-32B - revision: main - source: HF - max_seq_len: 40960 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 1 - runtime: - batch_scheduler_policy: max_utilization - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8/README.md deleted file mode 100644 index c8834a83e..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-8B-min-latency - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-8B-min-latency. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/Qwen3-8B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-8b-min-latency-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-8b-min-latency-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-8B - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 40960 - max_seq_len: 40960 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 32 - lookahead_verification_set_size: 1 - lookahead_windows_size: 1 - num_draft_tokens: 61 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8/README.md deleted file mode 100644 index 007468ceb..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# TensorRT-LLM Briton with Qwen/QwQ-32B-reasoning - -This is a Deployment for TensorRT-LLM Briton with Qwen/QwQ-32B-reasoning. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/QwQ-32B](https://huggingface.co/Qwen/QwQ-32B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/QwQ-32B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwq-32b-reasoning-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwq-32b-reasoning-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/QwQ-32B - revision: main - source: HF - max_seq_len: 40960 - num_builder_gpus: 4 - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp8 - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8/README.md deleted file mode 100644 index 41f705373..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8/README.md +++ /dev/null @@ -1,182 +0,0 @@ -# TensorRT-LLM Briton with Qwen/QwQ-32B-reasoning-with-speculative - -This is a Deployment for TensorRT-LLM Briton with Qwen/QwQ-32B-reasoning-with-speculative. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/QwQ-32B](https://huggingface.co/Qwen/QwQ-32B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -Qwen/QwQ-32B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwq-32b-reasoning-with-speculative-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwq-32b-reasoning-with-speculative-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/QwQ-32B - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 40960 - max_seq_len: 40960 - num_builder_gpus: 4 - quantization_config: - calib_max_seq_length: 2048 - calib_size: 2048 - quantization_type: fp8 - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 8 - lookahead_verification_set_size: 3 - lookahead_windows_size: 3 - num_draft_tokens: 41 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 1 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8/README.md b/11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8/README.md deleted file mode 100644 index dc5f1caa2..000000000 --- a/11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# TensorRT-LLM Briton with tiiuae/Falcon3-10B-Instruct - -This is a Deployment for TensorRT-LLM Briton with tiiuae/Falcon3-10B-Instruct. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [tiiuae/Falcon3-10B-Instruct](https://huggingface.co/tiiuae/Falcon3-10B-Instruct). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -tiiuae/Falcon3-10B-Instruct is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-tiiuae-falcon3-10b-instruct-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-tiiuae-falcon3-10b-instruct-fp8-truss-example -python_version: py39 -resources: - accelerator: L4:2 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: tiiuae/Falcon3-10B-Instruct - revision: main - source: HF - max_seq_len: 32768 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 2 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding/README.md deleted file mode 100644 index ea445cc61..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with Alibaba-NLP/gte-modernbert-base-embedding - -This is a Deployment for Huggingface's text-embeddings-inference with Alibaba-NLP/gte-modernbert-base-embedding. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Alibaba-NLP/gte-modernbert-base is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-alibaba-nlp-gte-modernbert-base-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: Alibaba-NLP/gte-modernbert-base - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-alibaba-nlp-gte-modernbert-base-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md deleted file mode 100644 index af1321660..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with Alibaba-NLP/gte-Qwen2-1.5B-instruct-embedding - -This is a Deployment for Huggingface's text-embeddings-inference with Alibaba-NLP/gte-Qwen2-1.5B-instruct-embedding. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Alibaba-NLP/gte-Qwen2-1.5B-instruct is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: Alibaba-NLP/gte-Qwen2-1.5B-instruct - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md deleted file mode 100644 index 06bdaf045..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with Alibaba-NLP/gte-Qwen2-7B-instruct-embedding - -This is a Deployment for Huggingface's text-embeddings-inference with Alibaba-NLP/gte-Qwen2-7B-instruct-embedding. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Alibaba-NLP/gte-Qwen2-7B-instruct is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:hopper-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: Alibaba-NLP/gte-Qwen2-7B-instruct - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base/README.md deleted file mode 100644 index 2b6005a5c..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base/README.md +++ /dev/null @@ -1,209 +0,0 @@ -# Huggingface's text-embeddings-inference with Alibaba-NLP/gte-reranker-modernbert-base - -This is a Deployment for Huggingface's text-embeddings-inference with Alibaba-NLP/gte-reranker-modernbert-base. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Alibaba-NLP/gte-reranker-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Reranker models may have at most one label, which contains the score of the reranking. - -Alibaba-NLP/gte-reranker-modernbert-base is a reranker model, used to re-rank a list of items, given a query. \nIt is frequently used in search engines, recommendation systems, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-alibaba-nlp-gte-reranker-modernbert-base-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank`: -```json -{ - "query": "What is Baseten?", - "raw_scores": true, - "return_text": false, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -response = client.rerank( - query="What is Baseten?", - texts=["Deep Learning is ...", "Baseten is a fast inference provider"], - raw_scores=True, - return_text=False, - truncate=True, -) -print(response.data) -``` - -Sometimes, you may want to apply a custom template to the texts before reranking them and call the predict endpoint instead: - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - # Custom template function to apply to the texts - # a popular template might be "{query}\n{document}" - # or also chat-style templates like "User: {query}\nDocument: {document}" - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["What is baseten? A: Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - - -### Requests python library - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank", - json={ - "query": "What is Baseten?", - "raw_scores": True, - "return_text": False, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": True, - "truncation_direction": "Right" -} -``` -Returns: -```json -[ - { - "index": 0, - "score": 1, - "text": "Deep Learning is ..." - } -] -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/rerank` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI.com does not have a rerank endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /rerank - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: Alibaba-NLP/gte-reranker-modernbert-base - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - query: What is Baseten? - raw_scores: true - return_text: true - texts: - - Deep Learning is ... - - Baseten is a fast inference provider - truncate: true - truncation_direction: Right -model_name: TEI-alibaba-nlp-gte-reranker-modernbert-base-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large/README.md deleted file mode 100644 index 600b1a9d1..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large/README.md +++ /dev/null @@ -1,209 +0,0 @@ -# Huggingface's text-embeddings-inference with BAAI/bge-reranker-large - -This is a Deployment for Huggingface's text-embeddings-inference with BAAI/bge-reranker-large. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Reranker models may have at most one label, which contains the score of the reranking. - -BAAI/bge-reranker-large is a reranker model, used to re-rank a list of items, given a query. \nIt is frequently used in search engines, recommendation systems, and more. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-baai-bge-reranker-large-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank`: -```json -{ - "query": "What is Baseten?", - "raw_scores": true, - "return_text": false, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": true, - "truncation_direction": "Right" -} -``` - -### Baseten Performance Client - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -response = client.rerank( - query="What is Baseten?", - texts=["Deep Learning is ...", "Baseten is a fast inference provider"], - raw_scores=True, - return_text=False, - truncate=True, -) -print(response.data) -``` - -Sometimes, you may want to apply a custom template to the texts before reranking them and call the predict endpoint instead: - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -def template(text: list[str]) -> list[str]: - # Custom template function to apply to the texts - # a popular template might be "{query}\n{document}" - # or also chat-style templates like "User: {query}\nDocument: {document}" - apply = lambda x: f"Custom template: {x}" - return [apply(t) for t in text] - -response = client.predict( - inputs=template(["What is baseten? A: Baseten is a fast inference provider", "Classify this separately."]), - raw_scores=True, - truncate=True, -) -print(response.data) -``` - - -### Requests python library - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/rerank", - json={ - "query": "What is Baseten?", - "raw_scores": True, - "return_text": False, - "texts": [ - "Deep Learning is ...", "Baseten is a fast inference provider" - ], - "truncate": True, - "truncation_direction": "Right" -} -``` -Returns: -```json -[ - { - "index": 0, - "score": 1, - "text": "Deep Learning is ..." - } -] -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/rerank` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI.com does not have a rerank endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:hopper-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /rerank - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: BAAI/bge-reranker-large - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - query: What is Baseten? - raw_scores: true - return_text: true - texts: - - Deep Learning is ... - - Baseten is a fast inference provider - truncate: true - truncation_direction: Right -model_name: TEI-baai-bge-reranker-large-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m/README.md deleted file mode 100644 index 58ed299cb..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with google/embeddinggemma-300m - -This is a Deployment for Huggingface's text-embeddings-inference with google/embeddinggemma-300m. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -google/embeddinggemma-300m is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-google-embeddinggemma-300m-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: google/embeddinggemma-300m - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-google-embeddinggemma-300m-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct/README.md deleted file mode 100644 index ca746073c..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with intfloat/multilingual-e5-large-instruct - -This is a Deployment for Huggingface's text-embeddings-inference with intfloat/multilingual-e5-large-instruct. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [intfloat/multilingual-e5-large-instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -intfloat/multilingual-e5-large-instruct is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-intfloat-multilingual-e5-large-instruct-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: intfloat/multilingual-e5-large-instruct - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-intfloat-multilingual-e5-large-instruct-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en/README.md deleted file mode 100644 index 6bad1f846..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with jina-ai/jina-embeddings-v2-base-en - -This is a Deployment for Huggingface's text-embeddings-inference with jina-ai/jina-embeddings-v2-base-en. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [jinaai/jina-embeddings-v2-base-en](https://huggingface.co/jinaai/jina-embeddings-v2-base-en). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -jinaai/jina-embeddings-v2-base-en is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-jina-ai-jina-embeddings-v2-base-en-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: jinaai/jina-embeddings-v2-base-en - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-jina-ai-jina-embeddings-v2-base-en-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code/README.md deleted file mode 100644 index d92a9723a..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with jinaai/jina-embeddings-v2-base-code - -This is a Deployment for Huggingface's text-embeddings-inference with jinaai/jina-embeddings-v2-base-code. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [jinaai/jina-embeddings-v2-base-code](https://huggingface.co/jinaai/jina-embeddings-v2-base-code). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -jinaai/jina-embeddings-v2-base-code is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-jinaai-jina-embeddings-v2-base-code-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: jinaai/jina-embeddings-v2-base-code - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-jinaai-jina-embeddings-v2-base-code-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md deleted file mode 100644 index 074002645..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with mixedbread-ai/mxbai-embed-large-v1-embedding - -This is a Deployment for Huggingface's text-embeddings-inference with mixedbread-ai/mxbai-embed-large-v1-embedding. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -mixedbread-ai/mxbai-embed-large-v1 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-mixedbread-ai-mxbai-embed-large-v1-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: mixedbread-ai/mxbai-embed-large-v1 - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-mixedbread-ai-mxbai-embed-large-v1-embedding-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5/README.md deleted file mode 100644 index 03a9dceb8..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with nomic-ai/nomic-embed-text-v1.5 - -This is a Deployment for Huggingface's text-embeddings-inference with nomic-ai/nomic-embed-text-v1.5. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -nomic-ai/nomic-embed-text-v1.5 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5 -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-nomic-ai-nomic-embed-text-v1.5-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:86-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: nomic-ai/nomic-embed-text-v1.5 - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-nomic-ai-nomic-embed-text-v1.5-truss-example -python_version: py39 -resources: - accelerator: A10G - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe/README.md deleted file mode 100644 index f98793c08..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with nomic-ai/nomic-embed-text-v2-moe - -This is a Deployment for Huggingface's text-embeddings-inference with nomic-ai/nomic-embed-text-v2-moe. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [nomic-ai/nomic-embed-text-v2-moe](https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -nomic-ai/nomic-embed-text-v2-moe is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-nomic-ai-nomic-embed-text-v2-moe-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: nomic-ai/nomic-embed-text-v2-moe - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-nomic-ai-nomic-embed-text-v2-moe-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2/README.md deleted file mode 100644 index 166582a3f..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with redis/langcache-embed-v2 - -This is a Deployment for Huggingface's text-embeddings-inference with redis/langcache-embed-v2. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [redis/langcache-embed-v2](https://huggingface.co/redis/langcache-embed-v2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -redis/langcache-embed-v2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2 -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-redis-langcache-embed-v2-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:89-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: redis/langcache-embed-v2 - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-redis-langcache-embed-v2-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding/README.md deleted file mode 100644 index 676b96dc6..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with sentence-transformers/all-MiniLM-L6-v2-embedding - -This is a Deployment for Huggingface's text-embeddings-inference with sentence-transformers/all-MiniLM-L6-v2-embedding. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -sentence-transformers/all-MiniLM-L6-v2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-sentence-transformers-all-minilm-l6-v2-embedding-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:turing-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: sentence-transformers/all-MiniLM-L6-v2 - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-sentence-transformers-all-minilm-l6-v2-embedding-truss-example -python_version: py39 -resources: - accelerator: T4 - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2/README.md b/11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2/README.md deleted file mode 100644 index 4facc5896..000000000 --- a/11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Huggingface's text-embeddings-inference with TaylorAI/bge-micro-v2 - -This is a Deployment for Huggingface's text-embeddings-inference with TaylorAI/bge-micro-v2. TEI is huggingface's solution for (text) embeddings, reranking models and prediction models. - -Supported models are tagged here: https://huggingface.co/models?other=text-embeddings-inference&sort=trending - -For TEI you have to perform a manual selection of the Docker Image. We have mirrored the following images: -``` -CPU baseten/text-embeddings-inference-mirror:cpu-1.8.3 -Turing (T4, ...) baseten/text-embeddings-inference-mirror:turing-1.8.3 -Ampere 80 (A100, A30) baseten/text-embeddings-inference-mirror:1.8.3 -Ampere 86 (A10, A10G, A40, ...) baseten/text-embeddings-inference-mirror:86-1.8.3 -Ada Lovelace (L4, ...) baseten/text-embeddings-inference-mirror:89-1.8.3 -Hopper (H100/H100 40GB/H200) baseten/text-embeddings-inference-mirror:hopper-1.8.3 -``` - -As we are deploying mostly tiny models (<1GB), we are downloading the model weights into the docker image. -For larger models, we recommend downloading the weights at runtime for faster autoscaling, as the weights don't need to go through decompression of the docker image. - - -# Examples: -This deployment is specifically designed for the Hugging Face model [TaylorAI/bge-micro-v2](https://huggingface.co/TaylorAI/bge-micro-v2). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -TaylorAI/bge-micro-v2 is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2 -``` - -With `11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model TEI-taylorai-bge-micro-v2-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### Baseten Performance Client - -```bash -pip install baseten-performance-client -``` - -```python -from baseten_performance_client import PerformanceClient - -client = PerformanceClient( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync" -) -texts = ["Hello world", "Example text", "Another sample"] -response = client.embed( - input=texts, - model="my_model", - batch_size=4, - max_concurrent_requests=32, - timeout_s=360, - # dimensions=1536 # optional for fp8 models. -) -print(response.numpy()) -``` - -Read more on the [Baseten Performance Client Blog](https://www.baseten.co/blog/your-client-code-matters-10x-higher-embedding-throughput-with-python-and-rust/) - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" - # dimensions=1536 # optional for MRL models. -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. - -```yaml -base_image: - image: baseten/text-embeddings-inference-mirror:86-1.8.3 -docker_server: - liveness_endpoint: /health - predict_endpoint: /v1/embeddings - readiness_endpoint: /health - server_port: 7997 - start_command: bash -c "truss-transfer-cli && text-embeddings-router --port 7997 - --model-id /app/model_cache/cached_model --max-client-batch-size 128 --max-concurrent-requests - 1024 --max-batch-tokens 16384 --auto-truncate --tokenization-workers 3" -model_cache: -- ignore_patterns: - - '*.pt' - - '*.ckpt' - - '*.onnx' - repo_id: TaylorAI/bge-micro-v2 - revision: main - use_volume: true - volume_folder: cached_model -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: TEI-taylorai-bge-micro-v2-truss-example -python_version: py39 -resources: - accelerator: A10G - cpu: '1' - memory: 2Gi - use_gpu: true -runtime: - is_websocket_endpoint: false - predict_concurrency: 32 - transport: - kind: http - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 528f049cf..ab45a3ce9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,98 +1,23 @@ # Contributing -Please open a PR to add a new model. Please add new models to the `model_library` folder. +We welcome new models and improvements to existing examples. Please open a PR. -## Style guide +## Adding a new example -Model library models should follow this style guide. +1. Place your example in the category that matches its type (`llm/`, `image/`, `audio/`, `embeddings/`, `infrastructure/`, or `tutorials/`). +2. Include a `config.yaml` with `model_name`, `description`, and `example_model_input`. +3. Include a `README.md` with deploy and invoke instructions. +4. Pin all Python requirements to specific versions. +5. Add your example path to `ci.yaml` at the repo root. -General style tips: +## Validate locally -* Truss folder name should use hyphens, not underscores (e.g. `falcon-7b` not `falcon_7b`) -* Truss folder name should not include the word Truss (e.g. `falcon-7b` not `falcon-7b-truss`) -* Model name should include parameter count when multiple variants exist (e.g. `Falcon 7B` not `Falcon`) - -## README - -The model should have a README that follows the layout of [Stable Diffusion XL](stable-diffusion/stable-diffusion-xl-1.0). - - -## Config - -Do not include any unnecessary or legacy config lines. - -Always include: - -* `model_name` -* `description` -* `model_metadata` - * `example_model_input` - -Optionally include: -* `model_metadata` - * `avatar_url` (Avatars/Logos should be 128x128 PNG) - * `cover_image_url` (Cover images should be 452x423 PNG) - * `tags` - -### Requirements - -Pin versions for all Python requirements! - -Example: - -```yaml -requirements: -- accelerate==0.20.3 -- bitsandbytes==0.39.1 -- peft==0.3.0 -- protobuf==4.23.3 -- sentencepiece==0.1.99 -- torch==2.0.1 -- transformers==4.30.2 -``` - -### Secrets - -Model library models can access secrets. - -If the model requires HuggingFace (e.g. Llama 2), always call the secret `hf_access_token` - -Example: - -```yaml -secrets: - hf_access_token: "ENTER HF ACCESS TOKEN HERE" +```bash +python _internal/bin/test_all.py ``` -### Hardware requirements - -Always configure a model library model with the least expensive hardware required to operate it at a reasonable degree of speed and quality. For example, Stable Diffusion XL defaults to an A10 even though performance is twice as fast on an A100. When these tradeoffs are made, note them in the README. - -## Model - -### Model I/O +This runs all config, README, naming, and CI checks. Pre-commit hooks run the same suite automatically. -* Models that support streaming should take a `stream` kwarg that defaults to false -* Models that take any kind of text input should call it `prompt` - -# Testing - -If you would like the model to be added the CI job that tests examples very day, add a reference -to the [ci.yaml](ci.yaml) file. - -# Automatic Documentation - -Some of the examples in this repo are used to generate automatic documentation on https://truss.baseten.co/. - -To add your model to this automatic documentation, add your example to one of the **top-level** categories -in the repo, and add a `doc.yaml` file that follows the following form: - -```yaml -title: "Text-to-image" -description: "Building a text-to-image model with SDXL" -files: - - model/model.py - - config.yaml -``` +## Questions? -See the [Introduction doc.yaml file](1_introduction/getting-started-bert/doc.yaml) for an example. +If your model doesn't fit an existing category, open an issue to discuss placement. diff --git a/README.md b/README.md index a878a90dc..3e204219e 100644 --- a/README.md +++ b/README.md @@ -2,41 +2,183 @@ [![Truss Examples CI](https://github.com/basetenlabs/truss-examples/actions/workflows/test-examples.yml/badge.svg)](https://github.com/basetenlabs/truss-examples/actions/workflows/test-examples.yml) -Truss is the simplest way to serve AI/ML models in production. +Production-ready **inference** examples for [Truss](https://truss.baseten.co/), +the simplest way to serve AI/ML models. Each example is ready to deploy as-is or +adapt to your own use case. -To get you started with [Truss](https://truss.baseten.co/), this repository has dozens of example models, each ready to deploy as-is or adapt to your needs. +> **Looking for training and fine-tuning?** See +> [ml-cookbook](https://github.com/basetenlabs/ml-cookbook) for training +> recipes, LoRA fine-tuning workflows, and end-to-end training examples on +> Baseten. -## Installation +## Quick start -Get the repository with: +Clone the repository: -``` +```bash git clone https://github.com/basetenlabs/truss-examples +cd truss-examples ``` -Install Truss with: +Install Truss: -``` +```bash pip install --upgrade truss ``` -## Deployment - - -Pick a model to deploy by passing a path to that model. +Deploy any example: ```bash -$ # From the truss-examples directory -$ truss push 02-llm +truss push tutorials/getting-started-bert ``` -This will prompt you for an API Key -- fetch one from the -[Baseten API keys page](https://app.baseten.co/settings/account/api_keys). - -## Invocation - -Invocation depends on the model's input and output specifications. See individual model READMEs for invocation details. - -# Contibuting - -We welcome contributions of new models and improvements to existing models. See [CONTRIBUTING.md](CONTRIBUTING.md) for details. +You will be prompted for an API key. Get one from the [Baseten API keys page](https://app.baseten.co/settings/account/api_keys). + +See individual example READMEs for invocation details specific to each model. + +## Repository structure + +| Category | Description | Examples | Path | +| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------- | ----------------- | +| [Tutorials](tutorials/) | Getting-started guides covering core Truss features: BERT, LLMs, streaming, image generation, speech-to-text, caching, dynamic batching, and more | 9 | `tutorials/` | +| [LLMs](llm/) | Large language model families including Llama, Qwen, DeepSeek, Mistral, Gemma, Phi, Falcon, and others | 19 families | `llm/` | +| [Embeddings](embeddings/) | Embedding and reranking models via BEI (48 models), TEI (15 models), CLIP, and text-embeddings-inference | 4 engines, 65 models | `embeddings/` | +| [Image](image/) | Image generation, editing, and segmentation: Stable Diffusion, Flux, SDXL, ComfyUI, ControlNet, SAM, and more | 15 | `image/` | +| [Audio](audio/) | Speech-to-text, text-to-speech, and audio generation: Whisper, Kokoro, Chatterbox, Orpheus, XTTS, MusicGen, and more | 19 | `audio/` | +| [Optimized](optimized/) | Production-grade optimized configs via Briton (33 TRT-LLM models) and BISv2 (11 models) | 2 engines, 44 models | `optimized/` | +| [Infrastructure](infrastructure/) | Patterns and techniques: custom servers, gRPC, Chains, multiprocessing, metrics, model caching, and more | 14 | `infrastructure/` | + +### Tutorials + +Step-by-step introductions to Truss concepts and features. + +| Example | Description | +| ------------------------------------------------------- | ------------------------------------------- | +| [getting-started-bert](tutorials/getting-started-bert/) | Deploy a BERT model for text classification | +| [llm-basics](tutorials/llm-basics/) | Serve an LLM with Truss | +| [llm-streaming](tutorials/llm-streaming/) | Stream LLM responses token-by-token | +| [image-generation](tutorials/image-generation/) | Serve an image generation model | +| [speech-to-text](tutorials/speech-to-text/) | Deploy a speech-to-text pipeline | +| [cached-weights](tutorials/cached-weights/) | Cache model weights for faster cold starts | +| [private-huggingface](tutorials/private-huggingface/) | Access private Hugging Face models | +| [dynamic-batching](tutorials/dynamic-batching/) | Enable dynamic batching for throughput | +| [system-packages](tutorials/system-packages/) | Add system-level dependencies | + +### LLMs + +Deploy popular large language model families with optimized serving configurations. + +| Model family | Path | +| ----------------- | -------------------------------------------------- | +| Llama | [`llm/llama/`](llm/llama/) | +| Qwen | [`llm/qwen/`](llm/qwen/) | +| DeepSeek | [`llm/deepseek/`](llm/deepseek/) | +| Mistral | [`llm/mistral/`](llm/mistral/) | +| Gemma | [`llm/gemma/`](llm/gemma/) | +| Phi | [`llm/phi/`](llm/phi/) | +| Falcon | [`llm/falcon/`](llm/falcon/) | +| Cogito | [`llm/cogito/`](llm/cogito/) | +| CogVLM | [`llm/cogvlm/`](llm/cogvlm/) | +| LLaVA | [`llm/llava/`](llm/llava/) | +| LoRA | [`llm/lora/`](llm/lora/) | +| OpenAI-compatible | [`llm/openai/`](llm/openai/) | +| Midnight | [`llm/midnight/`](llm/midnight/) | +| MiniMax | [`llm/minimax/`](llm/minimax/) | +| Nemotron | [`llm/nemotron/`](llm/nemotron/) | +| NSQL | [`llm/nsql/`](llm/nsql/) | +| Personaplex | [`llm/personaplex-7b-v1/`](llm/personaplex-7b-v1/) | +| Seed | [`llm/seed/`](llm/seed/) | +| Z-AI | [`llm/z-ai/`](llm/z-ai/) | + +### Embeddings + +Embedding, reranking, and classification models across multiple serving engines. + +| Engine | Models | Path | +| ------------------------- | ------ | -------------------------------------------------------------------------------- | +| BEI | 48 | [`embeddings/bei/`](embeddings/bei/) | +| TEI | 15 | [`embeddings/tei/`](embeddings/tei/) | +| CLIP | 1 | [`embeddings/clip/`](embeddings/clip/) | +| text-embeddings-inference | 1 | [`embeddings/text-embeddings-inference/`](embeddings/text-embeddings-inference/) | + +### Image + +Image generation, editing, upscaling, and segmentation models. + +| Example | Path | +| ------------------- | ------------------------------------------------------------------ | +| Stable Diffusion | [`image/stable-diffusion/`](image/stable-diffusion/) | +| Flux | [`image/flux/`](image/flux/) | +| Flux Dev TRT (B200) | [`image/flux-dev-trt-b200/`](image/flux-dev-trt-b200/) | +| Sana | [`image/sana/`](image/sana/) | +| ComfyUI | [`image/comfyui/`](image/comfyui/) | +| ControlNet QR Code | [`image/control-net-qrcode/`](image/control-net-qrcode/) | +| DeepFloyd XL | [`image/deepfloyd-xl/`](image/deepfloyd-xl/) | +| Fotographer | [`image/fotographer/`](image/fotographer/) | +| GFP-GAN | [`image/gfp-gan/`](image/gfp-gan/) | +| IP-Adapter | [`image/ip-adapter/`](image/ip-adapter/) | +| Magic Animate | [`image/magic-animate/`](image/magic-animate/) | +| Playground v2 | [`image/playground-v2-aesthetic/`](image/playground-v2-aesthetic/) | +| Segment Anything | [`image/segment-anything/`](image/segment-anything/) | +| DIS Segmentation | [`image/dis-segmentation/`](image/dis-segmentation/) | +| Image Segmentation | [`image/image-segmentation/`](image/image-segmentation/) | + +### Audio + +Speech-to-text, text-to-speech, and audio/music generation models. + +| Example | Path | +| -------------------------- | -------------------------------------------------------------------- | +| Whisper | [`audio/whisper/`](audio/whisper/) | +| Kokoro | [`audio/kokoro/`](audio/kokoro/) | +| Chatterbox TTS | [`audio/chatterbox-tts/`](audio/chatterbox-tts/) | +| Piper TTS | [`audio/piper-tts/`](audio/piper-tts/) | +| XTTS v2 | [`audio/xtts-v2/`](audio/xtts-v2/) | +| XTTS Streaming | [`audio/xtts-streaming/`](audio/xtts-streaming/) | +| Orpheus 3B (WebSockets) | [`audio/orpheus-3b-websockets/`](audio/orpheus-3b-websockets/) | +| Orpheus (Best Performance) | [`audio/orpheus-best-performance/`](audio/orpheus-best-performance/) | +| Sesame CSM 1B | [`audio/sesame-csm-1b/`](audio/sesame-csm-1b/) | +| MetaVoice 1B | [`audio/metavoice-1b/`](audio/metavoice-1b/) | +| Ultravox | [`audio/ultravox/`](audio/ultravox/) | +| AudioGen Medium | [`audio/audiogen-medium/`](audio/audiogen-medium/) | +| MusicGen Large | [`audio/musicgen-large/`](audio/musicgen-large/) | +| MusicGen Melody | [`audio/musicgen-melody/`](audio/musicgen-melody/) | +| NVIDIA Parakeet | [`audio/nvidia-parakeet/`](audio/nvidia-parakeet/) | +| Qwen ASR | [`audio/qwen-asr/`](audio/qwen-asr/) | +| Qwen Omni | [`audio/qwen-omni/`](audio/qwen-omni/) | +| Qwen Omni Thinker | [`audio/qwen-omni-thinker/`](audio/qwen-omni-thinker/) | +| Voxtral Streaming 4B | [`audio/voxtral-streaming-4b/`](audio/voxtral-streaming-4b/) | + +### Optimized + +Production-grade, autogenerated model configurations using TRT-LLM and other optimization engines. + +| Engine | Models | Path | +| ---------------- | ------ | ---------------------------------------- | +| Briton (TRT-LLM) | 33 | [`optimized/briton/`](optimized/briton/) | +| BISv2 | 11 | [`optimized/bisv2/`](optimized/bisv2/) | + +### Infrastructure + +Patterns, techniques, and advanced serving configurations. + +| Example | Path | +| ----------------------------- | ------------------------------------------------------------------------------------------------ | +| Custom Server | [`infrastructure/custom-server/`](infrastructure/custom-server/) | +| gRPC | [`infrastructure/grpc/`](infrastructure/grpc/) | +| Chains | [`infrastructure/chains-examples/`](infrastructure/chains-examples/) | +| Multiprocessing | [`infrastructure/multiprocessing/`](infrastructure/multiprocessing/) | +| Metrics | [`infrastructure/metrics/`](infrastructure/metrics/) | +| Model Cache | [`infrastructure/model-cache/`](infrastructure/model-cache/) | +| Custom Engine Builder Control | [`infrastructure/custom-engine-builder-control/`](infrastructure/custom-engine-builder-control/) | +| LLama.cpp Server | [`infrastructure/llama-cpp-server/`](infrastructure/llama-cpp-server/) | +| JSON Formatter | [`infrastructure/jsonformatter/`](infrastructure/jsonformatter/) | +| LayoutLM Document QA | [`infrastructure/layoutlm-document-qa/`](infrastructure/layoutlm-document-qa/) | +| N-gram Speculator | [`infrastructure/ngram-speculator/`](infrastructure/ngram-speculator/) | +| PaddlePaddle | [`infrastructure/paddlepaddle/`](infrastructure/paddlepaddle/) | +| Autodesk WALA | [`infrastructure/autodesk-wala/`](infrastructure/autodesk-wala/) | +| Binocular | [`infrastructure/binocular/`](infrastructure/binocular/) | + +## Contributing + +We welcome new models and improvements to existing examples. See [CONTRIBUTING.md](CONTRIBUTING.md) for details. diff --git a/_archive/README.md b/_archive/README.md new file mode 100644 index 000000000..c6a0c8ab5 --- /dev/null +++ b/_archive/README.md @@ -0,0 +1,11 @@ +# Archive + +These examples are deprecated or outdated. See the current examples in the parent directory. + +| Directory | Description | +|-----------|-------------| +| [deepspeed-mii](deepspeed-mii/) | DeepSpeed MII inference (deprecated) | +| [model_cach_gcs](model_cach_gcs/) | GCS model caching (deprecated) | +| [nous-capybara](nous-capybara/) | Nous Capybara model (deprecated) | +| [vllm](vllm/) | Legacy vLLM example (deprecated) | +| [whisper-torchserve](whisper-torchserve/) | Whisper speech-to-text via TorchServe backend (deprecated) | diff --git a/deepspeed-mii/README.md b/_archive/deepspeed-mii/README.md similarity index 100% rename from deepspeed-mii/README.md rename to _archive/deepspeed-mii/README.md diff --git a/deepspeed-mii/config.yaml b/_archive/deepspeed-mii/config.yaml similarity index 100% rename from deepspeed-mii/config.yaml rename to _archive/deepspeed-mii/config.yaml diff --git a/01-getting-started-bert/model/__init__.py b/_archive/deepspeed-mii/model/__init__.py similarity index 100% rename from 01-getting-started-bert/model/__init__.py rename to _archive/deepspeed-mii/model/__init__.py diff --git a/deepspeed-mii/model/model.py b/_archive/deepspeed-mii/model/model.py similarity index 100% rename from deepspeed-mii/model/model.py rename to _archive/deepspeed-mii/model/model.py diff --git a/model_cach_gcs/config.yaml b/_archive/model_cach_gcs/config.yaml similarity index 100% rename from model_cach_gcs/config.yaml rename to _archive/model_cach_gcs/config.yaml diff --git a/model_cach_gcs/model/model.py b/_archive/model_cach_gcs/model/model.py similarity index 100% rename from model_cach_gcs/model/model.py rename to _archive/model_cach_gcs/model/model.py diff --git a/nous-capybara/nous-capybara-34b-openai/README.md b/_archive/nous-capybara/nous-capybara-34b-openai/README.md similarity index 100% rename from nous-capybara/nous-capybara-34b-openai/README.md rename to _archive/nous-capybara/nous-capybara-34b-openai/README.md diff --git a/nous-capybara/nous-capybara-34b-openai/config.yaml b/_archive/nous-capybara/nous-capybara-34b-openai/config.yaml similarity index 100% rename from nous-capybara/nous-capybara-34b-openai/config.yaml rename to _archive/nous-capybara/nous-capybara-34b-openai/config.yaml diff --git a/02-llm/model/__init__.py b/_archive/nous-capybara/nous-capybara-34b-openai/model/__init__.py similarity index 100% rename from 02-llm/model/__init__.py rename to _archive/nous-capybara/nous-capybara-34b-openai/model/__init__.py diff --git a/nous-capybara/nous-capybara-34b-openai/model/model.py b/_archive/nous-capybara/nous-capybara-34b-openai/model/model.py similarity index 100% rename from nous-capybara/nous-capybara-34b-openai/model/model.py rename to _archive/nous-capybara/nous-capybara-34b-openai/model/model.py diff --git a/nous-capybara/nous-capybara-34b/README.md b/_archive/nous-capybara/nous-capybara-34b/README.md similarity index 100% rename from nous-capybara/nous-capybara-34b/README.md rename to _archive/nous-capybara/nous-capybara-34b/README.md diff --git a/nous-capybara/nous-capybara-34b/config.yaml b/_archive/nous-capybara/nous-capybara-34b/config.yaml similarity index 100% rename from nous-capybara/nous-capybara-34b/config.yaml rename to _archive/nous-capybara/nous-capybara-34b/config.yaml diff --git a/03-llm-with-streaming/model/__init__.py b/_archive/nous-capybara/nous-capybara-34b/model/__init__.py similarity index 100% rename from 03-llm-with-streaming/model/__init__.py rename to _archive/nous-capybara/nous-capybara-34b/model/__init__.py diff --git a/nous-capybara/nous-capybara-34b/model/model.py b/_archive/nous-capybara/nous-capybara-34b/model/model.py similarity index 100% rename from nous-capybara/nous-capybara-34b/model/model.py rename to _archive/nous-capybara/nous-capybara-34b/model/model.py diff --git a/vllm/README.md b/_archive/vllm/README.md similarity index 100% rename from vllm/README.md rename to _archive/vllm/README.md diff --git a/vllm/config.yaml b/_archive/vllm/config.yaml similarity index 100% rename from vllm/config.yaml rename to _archive/vllm/config.yaml diff --git a/04-image-generation/model/__init__.py b/_archive/vllm/model/__init__.py similarity index 100% rename from 04-image-generation/model/__init__.py rename to _archive/vllm/model/__init__.py diff --git a/phi/phi-3.5-mini/model/helper.py b/_archive/vllm/model/helper.py similarity index 100% rename from phi/phi-3.5-mini/model/helper.py rename to _archive/vllm/model/helper.py diff --git a/phi/phi-3.5-mini/model/model.py b/_archive/vllm/model/model.py similarity index 100% rename from phi/phi-3.5-mini/model/model.py rename to _archive/vllm/model/model.py diff --git a/whisper/whisper-torchserve/README.md b/_archive/whisper-torchserve/README.md similarity index 100% rename from whisper/whisper-torchserve/README.md rename to _archive/whisper-torchserve/README.md diff --git a/whisper/whisper-torchserve/config.yaml b/_archive/whisper-torchserve/config.yaml similarity index 100% rename from whisper/whisper-torchserve/config.yaml rename to _archive/whisper-torchserve/config.yaml diff --git a/whisper/whisper-torchserve/data/config.properties b/_archive/whisper-torchserve/data/config.properties similarity index 100% rename from whisper/whisper-torchserve/data/config.properties rename to _archive/whisper-torchserve/data/config.properties diff --git a/05-speech-to-text/model/__init__.py b/_archive/whisper-torchserve/model/__init__.py similarity index 100% rename from 05-speech-to-text/model/__init__.py rename to _archive/whisper-torchserve/model/__init__.py diff --git a/whisper/whisper-torchserve/model/model.py b/_archive/whisper-torchserve/model/model.py similarity index 100% rename from whisper/whisper-torchserve/model/model.py rename to _archive/whisper-torchserve/model/model.py diff --git a/11-embeddings-reranker-classification-tensorrt/README.md b/_internal/11-trt-original-README.md similarity index 100% rename from 11-embeddings-reranker-classification-tensorrt/README.md rename to _internal/11-trt-original-README.md diff --git a/_internal/README.md b/_internal/README.md new file mode 100644 index 000000000..0d0d4b0fe --- /dev/null +++ b/_internal/README.md @@ -0,0 +1,116 @@ +# _internal/ + +Development tooling, CI infrastructure, and codegen scripts for the truss-examples repository. Nothing in this directory is a user-facing example. + +## Directory layout + +``` +_internal/ +β”œβ”€β”€ bin/ # CLI scripts for testing, validation, and README generation +β”œβ”€β”€ templates/ # Legacy TRT-LLM / Whisper codegen templates +β”œβ”€β”€ templating/ # Programmatic config generation for BEI, TEI, Briton, BISV2 +β”œβ”€β”€ baseten-inference-stack-v2-templates/ # BIS V2 model configs (DeepSeek, Llama 4, Qwen, etc.) +β”œβ”€β”€ trt-llm-engine-builder-templates/ # TRT-LLM engine builder configs (Llama 3.1 variants) +β”œβ”€β”€ assets/ # Static assets (ComfyUI screenshots, workflow JSON) +β”œβ”€β”€ dockerfiles/ # Custom Dockerfiles (ComfyUI) +β”œβ”€β”€ essential/ # Config for Essential AI model (vLLM-based) +β”œβ”€β”€ internal/ # Config for internal Briton speculative decoding test +└── validation_report.json # Output from validate_all.py +``` + +## Scripts + +### bin/ + +| Script | Description | +|--------|-------------| +| `test_all.py` | Comprehensive local test suite: config validation, README checks, naming conventions, link validation, CI completeness | +| `validate_all.py` | Walks every config.yaml and produces a structured validation report (markdown, JSON, or CSV) | +| `validate_ci.py` | Loads every path listed in the top-level `ci.yaml` with `truss.load()` to verify CI paths are valid | +| `test_example.py` | CI script: detects changed model from git diff, pushes to staging, runs inference, cleans up old deployments | +| `test_truss_deploy.py` | Deploys a truss to staging via `truss push`, invokes inference with `example_model_input`, deactivates after | +| `generate_readmes.py` | Auto-generates standardized README.md files for every non-archived example from config.yaml metadata | +| `image.txt` | Base64-encoded test image used by `test_example.py` for image model inference | + +### templating/ + +| Script | Description | +|--------|-------------| +| `generate_templates.py` | Programmatically generates truss configs for BEI, TEI, Briton, and BIS V2 deployments from model definitions | +| `deploy_all.py` | Bulk deploy/delete/test operations for generated templates (supports `--action deploy\|delete\|britontest` with `--filter`) | + +### templates/ + +| File | Description | +|------|-------------| +| `generate.py` | Generates truss directories from base templates + config overrides defined in `generate.yaml` | +| `generate.yaml` | Declares legacy TRT-LLM and Transformers model variants (Mistral, Mixtral, Llama 2, Zephyr, Whisper) | +| `faster-whisper-truss/` | Base template for Faster Whisper models | +| `transformers-openai-compatible/` | Base template for HuggingFace Transformers with OpenAI-compatible API | +| `trt-llm/` | Base template for TRT-LLM models (includes Triton server configs) | + +## Running the test suite + +All scripts assume you are in the repository root. + +```sh +# Run all local validation checks +python _internal/bin/test_all.py + +# Verbose output (prints passing tests too) +python _internal/bin/test_all.py --verbose + +# Filter to a single category +python _internal/bin/test_all.py --category llm +``` + +The test suite runs 11 checks: + +1. Config validation (YAML parse + `truss.load()`) +2. README existence for every example +3. README-to-config consistency (endpoints, secrets, tags) +4. Directory naming conventions (hyphens, no underscores) +5. README link/path validation +6. `example_model_input` format validation +7. Requirements version pinning +8. CI path validation against `ci.yaml` +9. TRT-LLM OpenAI-compatible tag presence +10. `model.py` syntax validation (AST parse) +11. CI completeness (every example appears in `ci.yaml`) + +### Other validation commands + +```sh +# Structured validation report (writes validation_report.json) +python _internal/bin/validate_all.py +python _internal/bin/validate_all.py --json +python _internal/bin/validate_all.py --csv + +# Quick CI path check +python _internal/bin/validate_ci.py +``` + +### README generation + +```sh +# Regenerate all example READMEs from config.yaml metadata +python _internal/bin/generate_readmes.py +``` + +### Template generation (legacy) + +```sh +cd _internal/templates +python generate.py --root ../.. --templates . --config generate.yaml + +# Check-only mode (fails if generated output would differ) +python generate.py --only_check --root ../.. --templates . --config generate.yaml +``` + +## Dependencies + +- **truss** -- required by all scripts. Install with `uv pip install -e ../truss` or `pip install truss`. +- **PyYAML** -- used for config parsing (typically installed with truss). +- `templating/generate_templates.py` additionally requires `transformers`, `pydantic`, and `requests`. +- `templating/deploy_all.py` requires `BASETEN_API_KEY` environment variable and optionally `openai`. +- `templates/generate.py` requires `jinja2` and `pydantic`. diff --git a/assets/comfyui-screenshot-1.png b/_internal/assets/comfyui-screenshot-1.png similarity index 100% rename from assets/comfyui-screenshot-1.png rename to _internal/assets/comfyui-screenshot-1.png diff --git a/assets/comfyui-screenshot-2.png b/_internal/assets/comfyui-screenshot-2.png similarity index 100% rename from assets/comfyui-screenshot-2.png rename to _internal/assets/comfyui-screenshot-2.png diff --git a/assets/comfyui-screenshot-3.png b/_internal/assets/comfyui-screenshot-3.png similarity index 100% rename from assets/comfyui-screenshot-3.png rename to _internal/assets/comfyui-screenshot-3.png diff --git a/assets/sdxl-controlnet-workflow.json b/_internal/assets/sdxl-controlnet-workflow.json similarity index 100% rename from assets/sdxl-controlnet-workflow.json rename to _internal/assets/sdxl-controlnet-workflow.json diff --git a/baseten-inference-stack-v2-templates/deepseek-v3-0324/README.md b/_internal/baseten-inference-stack-v2-templates/deepseek-v3-0324/README.md similarity index 100% rename from baseten-inference-stack-v2-templates/deepseek-v3-0324/README.md rename to _internal/baseten-inference-stack-v2-templates/deepseek-v3-0324/README.md diff --git a/baseten-inference-stack-v2-templates/deepseek-v3-0324/config.yaml b/_internal/baseten-inference-stack-v2-templates/deepseek-v3-0324/config.yaml similarity index 100% rename from baseten-inference-stack-v2-templates/deepseek-v3-0324/config.yaml rename to _internal/baseten-inference-stack-v2-templates/deepseek-v3-0324/config.yaml diff --git a/baseten-inference-stack-v2-templates/glm47/config.yaml b/_internal/baseten-inference-stack-v2-templates/glm47/config.yaml similarity index 100% rename from baseten-inference-stack-v2-templates/glm47/config.yaml rename to _internal/baseten-inference-stack-v2-templates/glm47/config.yaml diff --git a/baseten-inference-stack-v2-templates/gpt-oss-120b/README.md b/_internal/baseten-inference-stack-v2-templates/gpt-oss-120b/README.md similarity index 100% rename from baseten-inference-stack-v2-templates/gpt-oss-120b/README.md rename to _internal/baseten-inference-stack-v2-templates/gpt-oss-120b/README.md diff --git a/baseten-inference-stack-v2-templates/gpt-oss-120b/config.yaml b/_internal/baseten-inference-stack-v2-templates/gpt-oss-120b/config.yaml similarity index 100% rename from baseten-inference-stack-v2-templates/gpt-oss-120b/config.yaml rename to _internal/baseten-inference-stack-v2-templates/gpt-oss-120b/config.yaml diff --git a/baseten-inference-stack-v2-templates/kimi-k2-instruct/README.md b/_internal/baseten-inference-stack-v2-templates/kimi-k2-instruct/README.md similarity index 100% rename from baseten-inference-stack-v2-templates/kimi-k2-instruct/README.md rename to _internal/baseten-inference-stack-v2-templates/kimi-k2-instruct/README.md diff --git a/baseten-inference-stack-v2-templates/kimi-k2-instruct/config.yaml b/_internal/baseten-inference-stack-v2-templates/kimi-k2-instruct/config.yaml similarity index 100% rename from baseten-inference-stack-v2-templates/kimi-k2-instruct/config.yaml rename to _internal/baseten-inference-stack-v2-templates/kimi-k2-instruct/config.yaml diff --git a/baseten-inference-stack-v2-templates/llama-4-maverick/config.yaml b/_internal/baseten-inference-stack-v2-templates/llama-4-maverick/config.yaml similarity index 100% rename from baseten-inference-stack-v2-templates/llama-4-maverick/config.yaml rename to _internal/baseten-inference-stack-v2-templates/llama-4-maverick/config.yaml diff --git a/baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/README.md b/_internal/baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/README.md similarity index 100% rename from baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/README.md rename to _internal/baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/README.md diff --git a/baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/config.yaml b/_internal/baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/config.yaml similarity index 100% rename from baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/config.yaml rename to _internal/baseten-inference-stack-v2-templates/qwen3-Coder-30b-instruct/config.yaml diff --git a/_internal/bin/generate_readmes.py b/_internal/bin/generate_readmes.py new file mode 100644 index 000000000..e866c9567 --- /dev/null +++ b/_internal/bin/generate_readmes.py @@ -0,0 +1,639 @@ +#!/usr/bin/env python3 +"""Generate standardized README.md for every non-archived example directory.""" + +from __future__ import annotations + +import json +import os +import re +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +SKIP_DIRS = {"_archive", "_internal", ".git", ".github", ".venv", "__pycache__"} + + +# --------------------------------------------------------------------------- +# Metadata extraction helpers +# --------------------------------------------------------------------------- + +def extract_hf_id(config: dict) -> str | None: + # 1. model_metadata.repo_id + repo_id = config.get("model_metadata", {}).get("repo_id") + if repo_id: + return repo_id + + # 2. trt_llm checkpoint repo + trt = config.get("trt_llm", {}) + repo = trt.get("build", {}).get("checkpoint_repository", {}).get("repo") + if repo: + return repo + + # 3. model_cache first entry + caches = config.get("model_cache", []) + if caches and isinstance(caches, list): + first = caches[0] if caches else {} + rid = first.get("repo_id") + if rid: + return rid + + # 4. Parse from docker_server.start_command + cmd = config.get("docker_server", {}).get("start_command", "") + for token in cmd.split(): + if "/" in token and not token.startswith("-") and not token.startswith("/"): + # Looks like org/model β€” strip any trailing punctuation + cleaned = token.strip("\"'") + if re.match(r"^[\w.-]+/[\w.-]+", cleaned): + return cleaned + + return None + + +def detect_engine(config: dict) -> str: + base_image = str(config.get("base_image", {}).get("image", "") if isinstance(config.get("base_image"), dict) else "") + start_cmd = config.get("docker_server", {}).get("start_command", "") + requirements = [str(r) for r in config.get("requirements", [])] + req_str = " ".join(requirements) + + if config.get("trt_llm"): + build = config["trt_llm"].get("build", {}) + base_model = build.get("base_model", "") + if base_model in ("encoder", "encoder_bert"): + return "BEI (TensorRT)" + return "TRT-LLM" + if "vllm" in base_image.lower() or "vllm" in start_cmd.lower(): + return "vLLM" + if "sglang" in start_cmd.lower() or "sglang" in req_str.lower(): + return "SGLang" + if "text-embeddings" in base_image.lower() or "text-embeddings-router" in start_cmd.lower(): + return "TEI (HuggingFace)" + if config.get("docker_server"): + return "Docker Server" + return "Custom (Truss)" + + +def infer_task_type(config: dict, category: str, dir_name: str) -> str: + tags = config.get("model_metadata", {}).get("tags", []) + if not isinstance(tags, list): + tags = [] + + # Check tags first + for tag in tags: + tag_l = tag.lower() + if "text-to-speech" in tag_l or "tts" in tag_l: + return "Text-to-speech" + if "speech-to-text" in tag_l or "stt" in tag_l or "transcription" in tag_l: + return "Speech-to-text" + if "image-generation" in tag_l: + return "Image generation" + if "embedding" in tag_l: + return "Embeddings" + if "rerank" in tag_l: + return "Reranking" + if "classification" in tag_l: + return "Classification" + + # Infer from category path + cat_l = category.lower() + if "llm" in cat_l or "optimized" in cat_l: + return "Text generation" + if "embedding" in cat_l: + # Check if it's a reranker + if "rerank" in dir_name.lower(): + return "Reranking" + if "classification" in dir_name.lower() or "reward" in dir_name.lower() or "ner" in dir_name.lower(): + return "Classification" + return "Embeddings" + if "image" in cat_l: + return "Image generation" + if "audio" in cat_l: + if "tts" in dir_name.lower() or "voice" in dir_name.lower() or "speech" in dir_name.lower() or "kokoro" in dir_name.lower() or "chatterbox" in dir_name.lower() or "metavoice" in dir_name.lower() or "sesame" in dir_name.lower(): + return "Text-to-speech" + if "whisper" in dir_name.lower() or "transcri" in dir_name.lower(): + return "Speech-to-text" + if "music" in dir_name.lower() or "audiogen" in dir_name.lower(): + return "Audio generation" + if "ultravox" in dir_name.lower(): + return "Audio understanding" + return "Audio" + if "infrastructure" in cat_l: + return "Infrastructure / Custom server" + if "tutorial" in cat_l: + return "Tutorial" + return "ML inference" + + +def extract_quantization(config: dict, dir_name: str) -> str | None: + # From config + trt = config.get("trt_llm", {}) + qt = trt.get("build", {}).get("quantization_type") + if qt: + return qt.upper().replace("_", " ") + + # From directory name + name_l = dir_name.lower() + for pat, label in [ + ("fp4", "FP4"), + ("fp8", "FP8"), + ("int8", "INT8"), + ("int4", "INT4"), + ("awq", "AWQ"), + ("gptq", "GPTQ"), + ("bnb", "BnB 4-bit"), + ]: + if pat in name_l: + return label + return None + + +def extract_gpu(config: dict) -> str: + acc = config.get("resources", {}).get("accelerator") + if acc: + return str(acc) + if config.get("resources", {}).get("use_gpu"): + return "GPU (unspecified)" + return "CPU" + + +def requires_hf_token(config: dict) -> bool: + secrets = config.get("secrets", {}) + if isinstance(secrets, dict) and "hf_access_token" in secrets: + return True + env = config.get("environment_variables", {}) + if isinstance(env, dict) and "hf_access_token" in env: + return True + return False + + +def get_api_endpoint(config: dict) -> str: + ep = config.get("docker_server", {}).get("predict_endpoint") + if ep: + return ep + rt = config.get("trt_llm", {}).get("runtime", {}) + route = rt.get("webserver_default_route") + if route: + return route + # OpenAI-compatible TRT-LLM defaults + tags = config.get("model_metadata", {}).get("tags", []) + if isinstance(tags, list) and "openai-compatible" in tags: + return "/v1/chat/completions" + return "/predict" + + +def is_openai_compatible(config: dict) -> bool: + tags = config.get("model_metadata", {}).get("tags", []) + if not isinstance(tags, list): + return False + return "openai-compatible" in tags + + +def get_category(example_dir: Path) -> str: + """Return the top-level category (llm, embeddings, image, audio, etc.).""" + rel = example_dir.relative_to(REPO_ROOT) + parts = rel.parts + if len(parts) >= 1: + return parts[0] + return "unknown" + + +def get_subcategory(example_dir: Path) -> str: + """Return deeper subcategory path for richer context.""" + rel = example_dir.relative_to(REPO_ROOT) + parts = rel.parts + if len(parts) >= 2: + return "/".join(parts[:2]) + return parts[0] if parts else "unknown" + + +# --------------------------------------------------------------------------- +# Config highlights +# --------------------------------------------------------------------------- + +def build_config_highlights(config: dict, engine: str) -> list[str]: + highlights = [] + + # Quantization + trt = config.get("trt_llm", {}) + build = trt.get("build", {}) + qt = build.get("quantization_type") + if qt: + highlights.append(f"Quantization: **{qt}**") + + # Tensor parallelism + tp = build.get("tensor_parallel_count") + if tp and tp > 1: + highlights.append(f"Tensor parallelism: **{tp}** GPUs") + + # Speculative decoding + spec = build.get("speculator", {}) + if spec: + mode = spec.get("speculative_decoding_mode", "enabled") + highlights.append(f"Speculative decoding: **{mode}**") + + # Max sequence length + max_seq = build.get("max_seq_len") + if max_seq: + highlights.append(f"Max sequence length: **{max_seq:,}**") + + # Chunked context + if trt.get("runtime", {}).get("enable_chunked_context"): + highlights.append("Chunked context: **enabled**") + + # Batch scheduler policy + bsp = trt.get("runtime", {}).get("batch_scheduler_policy") + if bsp: + highlights.append(f"Batch scheduler policy: **{bsp}**") + + # Plugin configuration + plugins = build.get("plugin_configuration", {}) + if plugins: + for k, v in plugins.items(): + if v: + highlights.append(f"Plugin: **{k}**") + + # Custom base image + base_image = config.get("base_image", {}) + if isinstance(base_image, dict) and base_image.get("image"): + highlights.append(f"Base image: `{base_image['image']}`") + + # Model cache / volume mounting + caches = config.get("model_cache", []) + if caches and isinstance(caches, list): + for c in caches: + if c.get("use_volume"): + highlights.append("Model cache: **volume-mounted** for fast cold starts") + break + + # Concurrency + conc = config.get("runtime", {}).get("predict_concurrency") + if conc: + highlights.append(f"Predict concurrency: **{conc}**") + + # System packages + sys_pkgs = config.get("system_packages", []) + if sys_pkgs: + highlights.append(f"System packages: `{', '.join(sys_pkgs)}`") + + # Streaming via example input + example = config.get("model_metadata", {}).get("example_model_input", {}) + if isinstance(example, dict) and example.get("stream"): + highlights.append("Streaming: **enabled**") + + # Environment variables (non-secret) + envs = config.get("environment_variables", {}) + if isinstance(envs, dict): + notable = {k: v for k, v in envs.items() if k != "hf_access_token" and v is not None} + if notable: + highlights.append(f"Environment variables: {', '.join(f'`{k}`' for k in notable)}") + + if not highlights: + highlights.append(f"Engine: **{engine}**") + + return highlights + + +# --------------------------------------------------------------------------- +# Invoke section +# --------------------------------------------------------------------------- + +def build_invoke_section( + config: dict, + engine: str, + task: str, + endpoint: str, + hf_id: str | None, + openai_compat: bool, +) -> str: + example_input = config.get("model_metadata", {}).get("example_model_input") + + # --- OpenAI-compatible LLM --- + if openai_compat and task == "Text generation": + model_name = hf_id or "model" + return f"""\ +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="{model_name}", + messages=[{{"role": "user", "content": "What is machine learning?"}}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{{"model": "{model_name}", "messages": [{{"role": "user", "content": "What is machine learning?"}}], "max_tokens": 512}}' +```""" + + # --- BEI / TEI embeddings --- + if endpoint == "/v1/embeddings": + model_name = hf_id or "model" + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{{"input": "What is deep learning?", "model": "{model_name}"}}' +```""" + + # --- Reranker --- + if endpoint == "/rerank" or task == "Reranking": + return """\ +```sh +curl -X POST https://model-.api.baseten.co/rerank \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +```""" + + # --- Use example_model_input if available --- + if example_input: + if isinstance(example_input, str): + try: + formatted = example_input + except Exception: + formatted = example_input + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co{endpoint} \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{formatted}' +```""" + else: + formatted = json.dumps(example_input, indent=2) + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co{endpoint} \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{formatted}' +```""" + + # --- Fallback by task --- + if task == "Text generation": + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co{endpoint} \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{{"prompt": "What is machine learning?", "max_tokens": 512}}' +```""" + + if task == "Image generation": + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co{endpoint} \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{{"prompt": "A photo of a cat in a field of sunflowers"}}' +``` + +> The response may contain base64-encoded image data.""" + + if task in ("Text-to-speech", "Audio generation"): + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co{endpoint} \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{{"text": "Hello, this is a test of text to speech."}}' +```""" + + if task == "Speech-to-text": + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co{endpoint} \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{{"url": "https://example.com/audio.wav"}}' +```""" + + # Generic fallback + return f"""\ +```sh +curl -X POST https://model-.api.baseten.co{endpoint} \\ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{{}}' +```""" + + +# --------------------------------------------------------------------------- +# Description generation +# --------------------------------------------------------------------------- + +def build_description(model_name: str, config: dict, category: str, engine: str, hf_id: str | None, task: str) -> str: + desc = config.get("description") + if desc: + return desc + + hf_part = f"[{hf_id}](https://huggingface.co/{hf_id})" if hf_id else model_name + engine_article = "an" if engine[0] in "AEIOU" else "a" + + if task == "Text generation": + return f"Deploy {hf_part} for text generation using {engine_article} {engine} engine on Baseten." + if task == "Embeddings": + return f"Deploy {hf_part} for generating text embeddings using {engine_article} {engine} engine on Baseten." + if task == "Reranking": + return f"Deploy {hf_part} as a reranker using {engine_article} {engine} engine on Baseten." + if task == "Classification": + return f"Deploy {hf_part} for classification using {engine_article} {engine} engine on Baseten." + if task == "Image generation": + return f"Deploy {hf_part} for image generation on Baseten." + if task == "Text-to-speech": + return f"Deploy {hf_part} for text-to-speech on Baseten." + if task == "Speech-to-text": + return f"Deploy {hf_part} for speech-to-text transcription on Baseten." + if task == "Audio generation": + return f"Deploy {hf_part} for audio generation on Baseten." + if task == "Audio understanding": + return f"Deploy {hf_part} for audio understanding on Baseten." + if task == "Tutorial": + return f"A tutorial example showing how to deploy {model_name} on Baseten." + if task == "Infrastructure / Custom server": + return f"Deploy {hf_part} using a custom server configuration on Baseten." + + return f"Deploy {model_name} on Baseten using {engine_article} {engine} engine." + + +# --------------------------------------------------------------------------- +# README rendering +# --------------------------------------------------------------------------- + +def render_readme( + model_name: str, + description: str, + hf_id: str | None, + task: str, + engine: str, + gpu: str, + quantization: str | None, + openai_compat: bool, + hf_token: bool, + endpoint: str, + highlights: list[str], + invoke_section: str, + has_model_py: bool, + python_version: str | None, +) -> str: + lines = [] + lines.append(f"# {model_name}\n") + lines.append(f"{description}\n") + + # Properties table + lines.append("| Property | Value |") + lines.append("|----------|-------|") + if hf_id: + lines.append(f"| Model | [{hf_id}](https://huggingface.co/{hf_id}) |") + lines.append(f"| Task | {task} |") + lines.append(f"| Engine | {engine} |") + lines.append(f"| GPU | {gpu} |") + if quantization: + lines.append(f"| Quantization | {quantization} |") + if openai_compat: + lines.append("| OpenAI compatible | Yes |") + if python_version: + lines.append(f"| Python | {python_version} |") + lines.append("") + + # Deploy + lines.append("## Deploy\n") + if hf_token: + lines.append("> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying.\n") + lines.append("```sh\ntruss push\n```\n") + + # Invoke + lines.append("## Invoke\n") + lines.append(invoke_section) + lines.append("") + + # Config highlights + lines.append("## Configuration highlights\n") + for h in highlights: + lines.append(f"- {h}") + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def find_example_dirs() -> list[Path]: + """Find all directories containing config.yaml, excluding archive/internal.""" + examples = [] + for root, dirs, files in os.walk(REPO_ROOT): + # Prune skip dirs + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + if "config.yaml" in files: + examples.append(Path(root)) + return sorted(examples) + + +def process_example(example_dir: Path) -> dict: + """Process a single example directory and return metadata + generated README.""" + config_path = example_dir / "config.yaml" + with open(config_path) as f: + config = yaml.safe_load(f) or {} + + dir_name = example_dir.name + category = get_category(example_dir) + + model_name = config.get("model_name") or dir_name.replace("-", " ").replace("_", " ").title() + hf_id = extract_hf_id(config) + engine = detect_engine(config) + task = infer_task_type(config, category, dir_name) + gpu = extract_gpu(config) + quantization = extract_quantization(config, dir_name) + openai_compat = is_openai_compatible(config) + hf_token = requires_hf_token(config) + endpoint = get_api_endpoint(config) + python_version = config.get("python_version") + has_model_py = (example_dir / "model" / "model.py").exists() + highlights = build_config_highlights(config, engine) + + description = build_description(model_name, config, category, engine, hf_id, task) + invoke = build_invoke_section(config, engine, task, endpoint, hf_id, openai_compat) + + readme = render_readme( + model_name=model_name, + description=description, + hf_id=hf_id, + task=task, + engine=engine, + gpu=gpu, + quantization=quantization, + openai_compat=openai_compat, + hf_token=hf_token, + endpoint=endpoint, + highlights=highlights, + invoke_section=invoke, + has_model_py=has_model_py, + python_version=python_version, + ) + + return { + "dir": str(example_dir), + "model_name": model_name, + "hf_id": hf_id, + "engine": engine, + "task": task, + "readme": readme, + } + + +def main(): + examples = find_example_dirs() + print(f"Found {len(examples)} example directories\n") + + generated = 0 + missing_hf = [] + + for example_dir in examples: + try: + result = process_example(example_dir) + except Exception as e: + print(f" ERROR: {example_dir.relative_to(REPO_ROOT)}: {e}") + continue + + readme_path = example_dir / "README.md" + readme_path.write_text(result["readme"]) + generated += 1 + + rel = example_dir.relative_to(REPO_ROOT) + status = "ok" if result["hf_id"] else "no HF ID" + print(f" {rel} [{result['engine']}] β€” {status}") + + if not result["hf_id"]: + missing_hf.append(str(rel)) + + print(f"\n--- Summary ---") + print(f"Generated: {generated}/{len(examples)}") + print(f"Missing HuggingFace ID: {len(missing_hf)}") + if missing_hf: + print("Directories without HF ID:") + for d in missing_hf: + print(f" - {d}") + + +if __name__ == "__main__": + main() diff --git a/bin/image.txt b/_internal/bin/image.txt similarity index 100% rename from bin/image.txt rename to _internal/bin/image.txt diff --git a/_internal/bin/test_all.py b/_internal/bin/test_all.py new file mode 100644 index 000000000..6bb0e66fe --- /dev/null +++ b/_internal/bin/test_all.py @@ -0,0 +1,605 @@ +#!/usr/bin/env python3 +"""Comprehensive test suite for truss-examples repository. + +Runs all validation that can be done locally without deploying models: + 1. Config validation (YAML parsing, truss.load()) + 2. README existence and structure + 3. README ↔ config consistency (endpoints, secrets, model names) + 4. Directory naming conventions + 5. Link/path validation (README links point to real dirs) + 6. example_model_input format validation + 7. Requirements pinning check + 8. CI path validation + +Usage: + python _internal/bin/test_all.py + python _internal/bin/test_all.py --verbose + python _internal/bin/test_all.py --category llm +""" + +import os +import re +import sys +from pathlib import Path + +import yaml + +try: + import truss +except ImportError: + print("ERROR: truss not installed. Run: uv pip install -e ../truss", file=sys.stderr) + sys.exit(1) + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +VERBOSE = "--verbose" in sys.argv or "-v" in sys.argv +CATEGORY_FILTER = None +for i, arg in enumerate(sys.argv): + if arg == "--category" and i + 1 < len(sys.argv): + CATEGORY_FILTER = sys.argv[i + 1] + +SKIP_DIRS = { + ".git", ".github", "__pycache__", "node_modules", ".venv", "venv", + "templating", "assets", "bin", "dockerfiles", "packages", "sample_images", + "examples", # skip nested examples dirs for dir naming checks +} + +CUSTOMER_CATEGORIES = ["tutorials", "llm", "embeddings", "image", "audio", "optimized", "infrastructure"] + +# Counters +passed = 0 +failed = 0 +warnings = 0 +failures = [] + + +def log_pass(msg): + global passed + passed += 1 + if VERBOSE: + print(f" PASS {msg}") + + +def log_fail(msg): + global failed + failed += 1 + failures.append(msg) + print(f" FAIL {msg}") + + +def log_warn(msg): + global warnings + warnings += 1 + if VERBOSE: + print(f" WARN {msg}") + + +def find_all_configs(root: Path) -> list[Path]: + """Find all directories containing a config.yaml.""" + configs = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in {".git", ".github", "__pycache__", "node_modules", ".venv", "venv"}] + if "config.yaml" in filenames: + rel = Path(dirpath).relative_to(root) + # Skip _archive + if str(rel).startswith("_archive"): + continue + if CATEGORY_FILTER and not str(rel).startswith(CATEGORY_FILTER): + continue + configs.append(Path(dirpath)) + return sorted(configs) + + +# ─── Test 1: Config validation ─────────────────────────────────────────────── + +def test_config_validation(): + print("\n== Test 1: Config Validation (YAML + truss.load) ==") + configs = find_all_configs(REPO_ROOT) + print(f" Found {len(configs)} configs to validate") + + broken = [] + for config_dir in configs: + rel = config_dir.relative_to(REPO_ROOT) + config_path = config_dir / "config.yaml" + try: + raw = yaml.safe_load(config_path.read_text()) + except Exception as e: + log_fail(f"{rel}: YAML parse error: {e}") + broken.append(str(rel)) + continue + + try: + truss.load(str(config_dir)) + log_pass(f"{rel}: config loads") + except Exception as e: + err = str(e)[:150] + # Allow known internal/archived breakages + if str(rel).startswith("_internal") or "/_archive/" in str(rel): + log_warn(f"{rel}: truss.load() failed (internal/archived): {err}") + else: + log_fail(f"{rel}: truss.load() failed: {err}") + broken.append(str(rel)) + + return broken + + +# ─── Test 2: README existence ──────────────────────────────────────────────── + +def test_readme_existence(): + print("\n== Test 2: README Existence ==") + configs = find_all_configs(REPO_ROOT) + missing = [] + + for config_dir in configs: + rel = config_dir.relative_to(REPO_ROOT) + if str(rel).startswith("_internal") or "/_archive/" in str(rel) or str(rel).startswith("_archive"): + continue + # Skip _archive at any level + if "_archive" in rel.parts: + continue + readme = config_dir / "README.md" + if readme.exists(): + log_pass(f"{rel}: has README.md") + else: + log_fail(f"{rel}: missing README.md") + missing.append(str(rel)) + + # Check category READMEs + for cat in CUSTOMER_CATEGORIES: + cat_readme = REPO_ROOT / cat / "README.md" + if cat_readme.exists(): + log_pass(f"{cat}/README.md exists") + else: + log_fail(f"{cat}/README.md missing") + + return missing + + +# ─── Test 3: README ↔ Config consistency ───────────────────────────────────── + +def test_readme_config_consistency(): + print("\n== Test 3: README ↔ Config Consistency ==") + configs = find_all_configs(REPO_ROOT) + + for config_dir in configs: + rel = config_dir.relative_to(REPO_ROOT) + if str(rel).startswith("_internal"): + continue + readme_path = config_dir / "README.md" + config_path = config_dir / "config.yaml" + + if not readme_path.exists(): + continue + + try: + raw = yaml.safe_load(config_path.read_text()) + except Exception: + continue + + if not raw or not isinstance(raw, dict): + continue + + readme_text = readme_path.read_text() + + # Check: if config has hf_access_token secret, README should mention it + secrets = raw.get("secrets", {}) + if secrets and isinstance(secrets, dict) and "hf_access_token" in secrets: + if "hf_access_token" in readme_text or "HuggingFace access token" in readme_text: + log_pass(f"{rel}: README mentions HF token (config requires it)") + else: + log_warn(f"{rel}: config has hf_access_token but README doesn't mention it") + + # Check: endpoint consistency + docker_server = raw.get("docker_server", {}) + if isinstance(docker_server, dict): + predict_endpoint = docker_server.get("predict_endpoint", "") + if predict_endpoint and predict_endpoint != "/predict": + # README should reference this endpoint + if predict_endpoint in readme_text: + log_pass(f"{rel}: README uses correct endpoint {predict_endpoint}") + elif "/predict" in readme_text and predict_endpoint not in readme_text: + log_warn(f"{rel}: README uses /predict but config has {predict_endpoint}") + + # Check: OpenAI-compatible tag should have /v1/chat/completions in README + tags = [] + metadata = raw.get("model_metadata", {}) + if isinstance(metadata, dict): + tags = metadata.get("tags", []) + if isinstance(tags, list) and "openai-compatible" in tags: + if "/v1/chat/completions" in readme_text or "OpenAI" in readme_text: + log_pass(f"{rel}: OpenAI-compatible model has correct invoke style") + else: + log_warn(f"{rel}: tagged openai-compatible but README lacks /v1/chat/completions") + + +# ─── Test 4: Directory naming conventions ──────────────────────────────────── + +def test_directory_naming(): + print("\n== Test 4: Directory Naming Conventions ==") + + for cat in CUSTOMER_CATEGORIES: + cat_dir = REPO_ROOT / cat + if not cat_dir.exists(): + continue + for dirpath, dirnames, filenames in os.walk(cat_dir): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".")] + rel = Path(dirpath).relative_to(REPO_ROOT) + + # If this directory contains config.yaml, it's an example root. + # Don't check naming of its children (model/, data/, etc. are internal). + if "config.yaml" in filenames: + dirnames.clear() + continue + + for d in dirnames: + if d.startswith("_"): + continue # _archive is OK + + # Check underscores (allow go_emotions which is upstream model name) + if "_" in d and "go_emotions" not in d: + log_fail(f"{rel}/{d}: directory name contains underscore (use hyphens)") + + # Check for "truss" in name + if "truss" in d.lower() and d != "truss": # allow ngram-speculator/truss + log_warn(f"{rel}/{d}: directory name contains 'truss'") + + # Check PascalCase (first char uppercase suggests PascalCase) + if d[0].isupper() and "-" in d: + log_warn(f"{rel}/{d}: directory name uses PascalCase (prefer lowercase)") + + +# ─── Test 5: Link/path validation in READMEs ──────────────────────────────── + +def test_readme_links(): + print("\n== Test 5: README Link Validation ==") + + # Check root README + root_readme = REPO_ROOT / "README.md" + if root_readme.exists(): + text = root_readme.read_text() + # Find all relative links like [text](path/) + links = re.findall(r'\[.*?\]\(([^)]+)\)', text) + for link in links: + if link.startswith("http") or link.startswith("#"): + continue + # Strip trailing / + clean = link.rstrip("/") + target = REPO_ROOT / clean + if target.exists() or (REPO_ROOT / clean.split("/")[0]).exists(): + log_pass(f"README.md: link to {clean} is valid") + else: + log_fail(f"README.md: broken link to {clean}") + + # Check category READMEs + for cat in CUSTOMER_CATEGORIES: + cat_readme = REPO_ROOT / cat / "README.md" + if not cat_readme.exists(): + continue + text = cat_readme.read_text() + links = re.findall(r'\[.*?\]\(([^)]+)\)', text) + for link in links: + if link.startswith("http") or link.startswith("#"): + continue + clean = link.rstrip("/") + target = REPO_ROOT / cat / clean + if target.exists(): + log_pass(f"{cat}/README.md: link to {clean} is valid") + else: + log_fail(f"{cat}/README.md: broken link to {clean}") + + # Check CONTRIBUTING.md + contrib = REPO_ROOT / "CONTRIBUTING.md" + if contrib.exists(): + text = contrib.read_text() + # Find paths in the table (backtick-wrapped) + paths = re.findall(r'`([a-z][\w/-]+)`', text) + for p in paths: + if "/" in p and not p.endswith(".py") and not p.endswith(".yaml") and not p.endswith(".yml"): + target = REPO_ROOT / p + if target.exists(): + log_pass(f"CONTRIBUTING.md: path {p} exists") + elif not any(p.endswith(ext) for ext in [".yaml", ".py", ".md", ".json"]): + # Check if it exists as a subdir within any category + found = any((REPO_ROOT / cat / p).exists() for cat in CUSTOMER_CATEGORIES) + if found: + log_pass(f"CONTRIBUTING.md: path {p} exists (relative to category)") + else: + log_warn(f"CONTRIBUTING.md: path {p} may not exist") + + +# ─── Test 6: example_model_input validation ────────────────────────────────── + +def test_example_model_input(): + print("\n== Test 6: example_model_input Validation ==") + configs = find_all_configs(REPO_ROOT) + missing_count = 0 + + for config_dir in configs: + rel = config_dir.relative_to(REPO_ROOT) + if str(rel).startswith("_internal") or "_archive" in rel.parts: + continue + + config_path = config_dir / "config.yaml" + try: + raw = yaml.safe_load(config_path.read_text()) + except Exception: + continue + + if not raw or not isinstance(raw, dict): + continue + + metadata = raw.get("model_metadata", {}) + if not isinstance(metadata, dict): + continue + + example_input = metadata.get("example_model_input") + if example_input is None: + missing_count += 1 + log_warn(f"{rel}: missing example_model_input") + continue + + # Validate input format based on category + rel_str = str(rel) + if rel_str.startswith("llm/"): + # LLMs: prompt, messages, query, text, message, image_url, queries all valid + if isinstance(example_input, dict): + valid_keys = {"prompt", "messages", "query", "text", "message", "image_url", "queries", "model"} + if valid_keys & set(example_input.keys()): + log_pass(f"{rel}: example_model_input has valid LLM format") + else: + log_warn(f"{rel}: LLM example_model_input missing prompt/messages/query") + elif isinstance(example_input, str): + log_pass(f"{rel}: example_model_input is a string (direct input)") + + elif rel_str.startswith("embeddings/"): + # Embeddings: input (standard), query/texts (rerankers), text/inputs (classifiers/NER), url (CLIP) + if isinstance(example_input, dict): + valid_keys = {"input", "inputs", "encoding_format", "query", "texts", "text", "model", "sentences", "url"} + if valid_keys & set(example_input.keys()): + log_pass(f"{rel}: example_model_input has valid embedding format") + else: + log_warn(f"{rel}: embedding example_model_input may have wrong format") + elif isinstance(example_input, str): + log_pass(f"{rel}: example_model_input is a string (direct input)") + else: + log_warn(f"{rel}: embedding example_model_input may have wrong format") + + elif rel_str.startswith("image/"): + # Image: prompt, image, instances, workflow_values (comfyui), input_image, reference_image, image_url + if isinstance(example_input, dict): + valid_keys = {"prompt", "image", "instances", "workflow", "workflow_values", "url", "input_image", "reference_image", "bbox", "image_url", "text"} + if valid_keys & set(example_input.keys()): + log_pass(f"{rel}: example_model_input has valid image format") + else: + log_warn(f"{rel}: image example_model_input may have wrong format") + else: + log_warn(f"{rel}: image example_model_input may have wrong format") + + elif rel_str.startswith("audio/"): + if isinstance(example_input, dict): + log_pass(f"{rel}: example_model_input is a dict") + elif isinstance(example_input, str): + log_pass(f"{rel}: example_model_input is a string") + + else: + log_pass(f"{rel}: has example_model_input") + + if missing_count: + print(f" {missing_count} configs still missing example_model_input") + + +# ─── Test 7: Requirements pinning ──────────────────────────────────────────── + +def test_requirements_pinning(): + print("\n== Test 7: Requirements Pinning ==") + configs = find_all_configs(REPO_ROOT) + unpinned_count = 0 + + for config_dir in configs: + rel = config_dir.relative_to(REPO_ROOT) + if str(rel).startswith("_internal") or str(rel).startswith("_archive"): + continue + + config_path = config_dir / "config.yaml" + try: + raw = yaml.safe_load(config_path.read_text()) + except Exception: + continue + + if not raw or not isinstance(raw, dict): + continue + + requirements = raw.get("requirements", []) + if not isinstance(requirements, list): + continue + + # Check if using requirements_file instead + if raw.get("requirements_file"): + continue + + unpinned = [] + for req in requirements: + if not isinstance(req, str): + continue + req = req.strip() + if not req or req.startswith("#") or req.startswith("-"): + continue + if req.startswith("git+"): + # git+ URLs should pin to a commit hash, not @master or @main + if "@master" in req or "@main" in req: + unpinned.append(req) + continue + # Check for version pin + if "==" not in req and ">=" not in req and "<=" not in req and "~=" not in req: + unpinned.append(req) + + if unpinned: + unpinned_count += 1 + for u in unpinned: + log_warn(f"{rel}: unpinned requirement: {u}") + elif requirements: + log_pass(f"{rel}: all requirements pinned") + + if unpinned_count: + print(f" {unpinned_count} configs have unpinned requirements") + + +# ─── Test 8: CI path validation ────────────────────────────────────────────── + +def test_ci_paths(): + print("\n== Test 8: CI Path Validation ==") + ci_path = REPO_ROOT / "ci.yaml" + if not ci_path.exists(): + log_fail("ci.yaml not found") + return + + with open(ci_path) as f: + ci = yaml.safe_load(f) + + tests = ci.get("tests", []) + print(f" {len(tests)} CI test paths") + + for test_path in tests: + full_path = REPO_ROOT / test_path + if full_path.exists(): + # Also verify it loads + try: + truss.load(str(full_path)) + log_pass(f"ci.yaml: {test_path} exists and loads") + except Exception as e: + log_fail(f"ci.yaml: {test_path} exists but truss.load() fails: {str(e)[:100]}") + else: + log_fail(f"ci.yaml: {test_path} does not exist") + + +# ─── Test 9: TRT-LLM openai-compatible tag ────────────────────────────────── + +def test_trt_llm_tags(): + print("\n== Test 9: TRT-LLM openai-compatible Tag ==") + configs = find_all_configs(REPO_ROOT) + missing = 0 + + for config_dir in configs: + rel = config_dir.relative_to(REPO_ROOT) + if "_archive" in rel.parts or "_internal" in rel.parts: + continue + + config_path = config_dir / "config.yaml" + try: + raw = yaml.safe_load(config_path.read_text()) + except Exception: + continue + + if not raw or not isinstance(raw, dict): + continue + + if not raw.get("trt_llm"): + continue + + tags = raw.get("model_metadata", {}).get("tags", []) + if not isinstance(tags, list): + tags = [] + + has_oai = "openai-compatible" in tags + has_legacy = "force-legacy-api-non-openai-compatible" in tags + if has_oai or has_legacy: + log_pass(f"{rel}: TRT-LLM has API compatibility tag") + else: + log_fail(f"{rel}: TRT-LLM missing openai-compatible or force-legacy tag") + missing += 1 + + if missing: + print(f" {missing} TRT-LLM configs missing API compatibility tag") + + +# ─── Test 10: model.py syntax validation ──────────────────────────────────── + +def test_model_py_syntax(): + print("\n== Test 10: model.py Syntax Validation ==") + import ast + + model_files = sorted(REPO_ROOT.glob("**/model/model.py")) + checked = 0 + + for mf in model_files: + rel = mf.relative_to(REPO_ROOT) + if "_archive" in rel.parts or "_internal" in rel.parts: + continue + checked += 1 + try: + ast.parse(mf.read_text()) + log_pass(f"{rel}: syntax OK") + except SyntaxError as e: + log_fail(f"{rel}: syntax error: {e}") + + print(f" Checked {checked} model.py files") + + +# ─── Test 10: CI completeness ─────────────────────────────────────────────── + +def test_ci_completeness(): + print("\n== Test 11: CI Completeness (ci.yaml covers all examples) ==") + ci_path = REPO_ROOT / "ci.yaml" + if not ci_path.exists(): + log_fail("ci.yaml not found") + return + + with open(ci_path) as f: + ci = yaml.safe_load(f) + + ci_paths = set(ci.get("tests", [])) + configs = find_all_configs(REPO_ROOT) + missing_from_ci = [] + + for config_dir in configs: + rel = config_dir.relative_to(REPO_ROOT) + rel_str = str(rel) + if rel_str.startswith("_internal") or "_archive" in rel.parts: + continue + if rel_str not in ci_paths: + missing_from_ci.append(rel_str) + log_fail(f"ci.yaml missing: {rel_str}") + else: + log_pass(f"ci.yaml covers: {rel_str}") + + if missing_from_ci: + print(f" {len(missing_from_ci)} examples not in ci.yaml") + else: + print(f" ci.yaml covers all {len(ci_paths)} examples") + + +# ─── Main ──────────────────────────────────────────────────────────────────── + +def main(): + print("=" * 60) + print(" Truss Examples - Comprehensive Test Suite") + print("=" * 60) + + test_config_validation() + test_readme_existence() + test_readme_config_consistency() + test_directory_naming() + test_readme_links() + test_example_model_input() + test_requirements_pinning() + test_ci_paths() + test_trt_llm_tags() + test_model_py_syntax() + test_ci_completeness() + + print("\n" + "=" * 60) + print(f" Results: {passed} passed, {failed} failed, {warnings} warnings") + print("=" * 60) + + if failures: + print(f"\n {len(failures)} failure(s):") + for f in failures: + print(f" - {f}") + print() + sys.exit(1) + else: + print("\n All tests passed!\n") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/bin/test_example.py b/_internal/bin/test_example.py similarity index 100% rename from bin/test_example.py rename to _internal/bin/test_example.py diff --git a/bin/test_truss_deploy.py b/_internal/bin/test_truss_deploy.py similarity index 98% rename from bin/test_truss_deploy.py rename to _internal/bin/test_truss_deploy.py index 33edb2938..081fadfd8 100644 --- a/bin/test_truss_deploy.py +++ b/_internal/bin/test_truss_deploy.py @@ -153,7 +153,7 @@ def get_time_in_ms(): if __name__ == "__main__": model_dir = get_model_dir() - image_str = open("bin/image.txt", "r").read() + image_str = open(os.path.join(os.path.dirname(__file__), "image.txt"), "r").read() os.chdir(model_dir) example_input = get_example_input(image_str) diff --git a/_internal/bin/validate_all.py b/_internal/bin/validate_all.py new file mode 100644 index 000000000..e0e6036f7 --- /dev/null +++ b/_internal/bin/validate_all.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Validate every truss example config in the repository. + +Walks the repo, finds every directory containing a config.yaml, +and runs validation checks. Outputs a markdown report to stdout. + +Usage: + python bin/validate_all.py [--json] [--csv] +""" + +import ast +import json +import os +import sys +from pathlib import Path + +import yaml + +# Ensure we can import truss +try: + import truss + from truss.base.truss_config import TrussConfig +except ImportError: + print("ERROR: truss not installed. Run: uv pip install -e ../truss", file=sys.stderr) + sys.exit(1) + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +# Directories to skip entirely (not examples) +SKIP_DIRS = { + ".git", + ".github", + "__pycache__", + "node_modules", + ".venv", + "venv", + "templating", + "assets", + "bin", + "dockerfiles", +} + +# Patterns that indicate a deprecated/archivable example +ARCHIVE_PATTERNS = [ + "llama-2-", + "llama-7b", + "mistral-7b-chat", + "mistral-7b-instruct", + "whisper-torchserve", + "deepspeed-mii", + "nous-capybara", + "model_cach_gcs", + "qwen-7b-chat", +] + +# Deprecated config fields +DEPRECATED_FIELDS_IN_YAML = ["hf_cache"] + + +class ValidationResult: + def __init__(self, path: str): + self.path = path + self.status = "VALID" + self.issues: list[str] = [] + self.warnings: list[str] = [] + self.has_model_py = False + self.has_readme = False + self.has_example_input = False + self.has_docker_server = False + self.has_trt_llm = False + self.config_loads = False + + def add_issue(self, msg: str): + self.issues.append(msg) + + def add_warning(self, msg: str): + self.warnings.append(msg) + + def determine_status(self): + if not self.config_loads: + self.status = "BROKEN" + return + + # Check archive patterns + dir_name = Path(self.path).name.lower() + for pattern in ARCHIVE_PATTERNS: + if pattern in dir_name: + self.status = "ARCHIVE" + return + + if self.issues: + self.status = "DEPRECATED" + return + + if not self.has_example_input and not self.has_readme: + self.status = "VALID_NO_INPUT" + elif not self.has_readme: + self.status = "VALID_NO_README" + elif not self.has_example_input: + self.status = "VALID_NO_INPUT" + else: + self.status = "VALID" + + +def find_all_configs(root: Path) -> list[Path]: + """Find all directories containing a config.yaml.""" + configs = [] + for dirpath, dirnames, filenames in os.walk(root): + # Prune skip dirs + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + if "config.yaml" in filenames: + configs.append(Path(dirpath)) + return sorted(configs) + + +def check_model_py(example_dir: Path) -> tuple[bool, list[str]]: + """Check if model.py exists and has load/predict methods via AST.""" + issues = [] + model_dir = example_dir / "model" + model_py = model_dir / "model.py" + + if not model_py.exists(): + # Also check root level + model_py = example_dir / "model.py" + + if not model_py.exists(): + return False, [] + + try: + source = model_py.read_text() + tree = ast.parse(source) + except SyntaxError as e: + return True, [f"model.py has syntax error: {e}"] + + # Find class with load/predict + has_load = False + has_predict = False + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + for item in node.body: + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + if item.name == "load": + has_load = True + if item.name == "predict": + has_predict = True + + if not has_load: + issues.append("model.py: no load() method found") + if not has_predict: + issues.append("model.py: no predict() method found") + + return True, issues + + +def check_config_yaml(example_dir: Path) -> ValidationResult: + """Run all validation checks on a single example.""" + result = ValidationResult(str(example_dir.relative_to(REPO_ROOT))) + config_path = example_dir / "config.yaml" + + # 1. Try loading with truss + try: + raw = yaml.safe_load(config_path.read_text()) + except Exception as e: + result.add_issue(f"YAML parse error: {e}") + result.determine_status() + return result + + # Check for deprecated fields in raw YAML + if raw and isinstance(raw, dict): + for field in DEPRECATED_FIELDS_IN_YAML: + if field in raw: + result.add_issue(f"Deprecated field '{field}' found in config") + + # Check for old base_model values + trt_llm = raw.get("trt_llm", {}) + if isinstance(trt_llm, dict): + build = trt_llm.get("build", {}) + if isinstance(build, dict): + base_model = build.get("base_model", "") + if base_model in ("llama", "mistral"): + result.add_issue( + f"Deprecated base_model value '{base_model}' (use 'decoder')" + ) + + # Try truss.load() + try: + _ = truss.load(str(example_dir)) + result.config_loads = True + except Exception as e: + err_str = str(e) + # Truncate long errors + if len(err_str) > 200: + err_str = err_str[:200] + "..." + result.add_issue(f"truss.load() failed: {err_str}") + result.config_loads = False + result.determine_status() + return result + + # 2. Check for docker_server or trt_llm + if raw and isinstance(raw, dict): + result.has_docker_server = "docker_server" in raw and raw["docker_server"] + result.has_trt_llm = "trt_llm" in raw and raw["trt_llm"] + + # 3. Check model.py (not required for docker_server or trt_llm configs) + if not result.has_docker_server and not result.has_trt_llm: + has_model, model_issues = check_model_py(example_dir) + result.has_model_py = has_model + if not has_model: + result.add_warning("No model.py found") + for issue in model_issues: + result.add_warning(issue) + else: + result.has_model_py = True # Not applicable + + # 4. Check example_model_input + if raw and isinstance(raw, dict): + model_metadata = raw.get("model_metadata", {}) + if isinstance(model_metadata, dict): + result.has_example_input = "example_model_input" in model_metadata + + # 5. Check README.md + result.has_readme = (example_dir / "README.md").exists() + + # 6. Check requirements.txt parseable + req_file = example_dir / "requirements.txt" + if req_file.exists(): + try: + lines = req_file.read_text().strip().split("\n") + for line in lines: + line = line.strip() + if line and not line.startswith("#") and not line.startswith("-"): + # Basic check - should have valid pip format + pass + except Exception as e: + result.add_warning(f"requirements.txt issue: {e}") + + result.determine_status() + return result + + +def main(): + output_format = "markdown" + if "--json" in sys.argv: + output_format = "json" + elif "--csv" in sys.argv: + output_format = "csv" + + configs = find_all_configs(REPO_ROOT) + print(f"Found {len(configs)} example directories with config.yaml", file=sys.stderr) + + results: list[ValidationResult] = [] + for config_dir in configs: + result = check_config_yaml(config_dir) + results.append(result) + + # Summary counts + status_counts: dict[str, int] = {} + for r in results: + status_counts[r.status] = status_counts.get(r.status, 0) + 1 + + if output_format == "json": + data = [] + for r in results: + data.append( + { + "path": r.path, + "status": r.status, + "issues": r.issues, + "warnings": r.warnings, + "has_model_py": r.has_model_py, + "has_readme": r.has_readme, + "has_example_input": r.has_example_input, + "has_docker_server": r.has_docker_server, + "has_trt_llm": r.has_trt_llm, + } + ) + print(json.dumps({"summary": status_counts, "results": data}, indent=2)) + + elif output_format == "csv": + print("path,status,issues,warnings,has_model_py,has_readme,has_example_input") + for r in results: + issues = "; ".join(r.issues).replace(",", ";") + warnings = "; ".join(r.warnings).replace(",", ";") + print( + f"{r.path},{r.status},{issues},{warnings},{r.has_model_py},{r.has_readme},{r.has_example_input}" + ) + + else: + # Markdown + print("# Validation Report\n") + print(f"**Total examples**: {len(results)}\n") + print("## Summary\n") + print("| Status | Count |") + print("|--------|-------|") + for status in [ + "VALID", + "VALID_NO_INPUT", + "VALID_NO_README", + "DEPRECATED", + "BROKEN", + "ARCHIVE", + ]: + count = status_counts.get(status, 0) + if count > 0: + print(f"| {status} | {count} |") + print() + + # Group by status + for status in [ + "BROKEN", + "DEPRECATED", + "ARCHIVE", + "VALID_NO_INPUT", + "VALID_NO_README", + "VALID", + ]: + group = [r for r in results if r.status == status] + if not group: + continue + print(f"\n## {status} ({len(group)})\n") + print("| Path | Issues | Warnings |") + print("|------|--------|----------|") + for r in group: + issues = "; ".join(r.issues) if r.issues else "-" + warnings = "; ".join(r.warnings) if r.warnings else "-" + print(f"| `{r.path}` | {issues} | {warnings} |") + + # Write report to file as well + report_path = REPO_ROOT / "validation_report.json" + data = [] + for r in results: + data.append( + { + "path": r.path, + "status": r.status, + "issues": r.issues, + "warnings": r.warnings, + } + ) + report_path.write_text(json.dumps({"summary": status_counts, "results": data}, indent=2)) + print(f"\nReport also saved to {report_path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/bin/validate_ci.py b/_internal/bin/validate_ci.py similarity index 100% rename from bin/validate_ci.py rename to _internal/bin/validate_ci.py diff --git a/dockerfiles/ComfyUI.dockerfile b/_internal/dockerfiles/ComfyUI.dockerfile similarity index 100% rename from dockerfiles/ComfyUI.dockerfile rename to _internal/dockerfiles/ComfyUI.dockerfile diff --git a/essential/config.yaml b/_internal/essential/config.yaml similarity index 100% rename from essential/config.yaml rename to _internal/essential/config.yaml diff --git a/internal/config.yaml b/_internal/internal/config.yaml similarity index 100% rename from internal/config.yaml rename to _internal/internal/config.yaml diff --git a/templates/README.md b/_internal/templates/README.md similarity index 100% rename from templates/README.md rename to _internal/templates/README.md diff --git a/templates/faster-whisper-truss/config.yaml b/_internal/templates/faster-whisper-truss/config.yaml similarity index 100% rename from templates/faster-whisper-truss/config.yaml rename to _internal/templates/faster-whisper-truss/config.yaml diff --git a/06-high-performance-cached-weights/model/__init__.py b/_internal/templates/faster-whisper-truss/model/__init__.py similarity index 100% rename from 06-high-performance-cached-weights/model/__init__.py rename to _internal/templates/faster-whisper-truss/model/__init__.py diff --git a/templates/faster-whisper-truss/model/model.py b/_internal/templates/faster-whisper-truss/model/model.py similarity index 100% rename from templates/faster-whisper-truss/model/model.py rename to _internal/templates/faster-whisper-truss/model/model.py diff --git a/templates/generate.py b/_internal/templates/generate.py similarity index 100% rename from templates/generate.py rename to _internal/templates/generate.py diff --git a/templates/generate.yaml b/_internal/templates/generate.yaml similarity index 100% rename from templates/generate.yaml rename to _internal/templates/generate.yaml diff --git a/templates/transformers-openai-compatible/config.yaml b/_internal/templates/transformers-openai-compatible/config.yaml similarity index 100% rename from templates/transformers-openai-compatible/config.yaml rename to _internal/templates/transformers-openai-compatible/config.yaml diff --git a/07-high-performance-dynamic-batching/model/__init__.py b/_internal/templates/transformers-openai-compatible/model/__init__.py similarity index 100% rename from 07-high-performance-dynamic-batching/model/__init__.py rename to _internal/templates/transformers-openai-compatible/model/__init__.py diff --git a/templates/transformers-openai-compatible/model/model.py b/_internal/templates/transformers-openai-compatible/model/model.py similarity index 100% rename from templates/transformers-openai-compatible/model/model.py rename to _internal/templates/transformers-openai-compatible/model/model.py diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/TRT-LLM-README.md b/_internal/templates/trt-llm/TRT-LLM-README.md similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/TRT-LLM-README.md rename to _internal/templates/trt-llm/TRT-LLM-README.md diff --git a/templates/trt-llm/config.yaml b/_internal/templates/trt-llm/config.yaml similarity index 100% rename from templates/trt-llm/config.yaml rename to _internal/templates/trt-llm/config.yaml diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/data/.gitattributes b/_internal/templates/trt-llm/data/.gitattributes similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/data/.gitattributes rename to _internal/templates/trt-llm/data/.gitattributes diff --git a/07-high-performance-dynamic-batching/packages/__init__.py b/_internal/templates/trt-llm/model/__init__.py similarity index 100% rename from 07-high-performance-dynamic-batching/packages/__init__.py rename to _internal/templates/trt-llm/model/__init__.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/model/model.py b/_internal/templates/trt-llm/model/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/model/model.py rename to _internal/templates/trt-llm/model/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/client.py b/_internal/templates/trt-llm/packages/client.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/client.py rename to _internal/templates/trt-llm/packages/client.py diff --git a/templates/trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt.jinja b/_internal/templates/trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt.jinja similarity index 100% rename from templates/trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt.jinja rename to _internal/templates/trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt.jinja diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/1/model.py b/_internal/templates/trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/1/model.py rename to _internal/templates/trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py diff --git a/templates/trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt.jinja b/_internal/templates/trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt.jinja similarity index 100% rename from templates/trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt.jinja rename to _internal/templates/trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt.jinja diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/1/model.py b/_internal/templates/trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/1/model.py rename to _internal/templates/trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py diff --git a/templates/trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt.jinja b/_internal/templates/trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt.jinja similarity index 100% rename from templates/trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt.jinja rename to _internal/templates/trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt.jinja diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/_internal/templates/trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt rename to _internal/templates/trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/utils.py b/_internal/templates/trt-llm/packages/utils.py similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/utils.py rename to _internal/templates/trt-llm/packages/utils.py diff --git a/11-embeddings-reranker-classification-tensorrt/templating/.internal_tei/Dockerfile b/_internal/templating/.internal_tei/Dockerfile similarity index 100% rename from 11-embeddings-reranker-classification-tensorrt/templating/.internal_tei/Dockerfile rename to _internal/templating/.internal_tei/Dockerfile diff --git a/11-embeddings-reranker-classification-tensorrt/templating/.internal_tei/roll_out_docker.sh b/_internal/templating/.internal_tei/roll_out_docker.sh similarity index 100% rename from 11-embeddings-reranker-classification-tensorrt/templating/.internal_tei/roll_out_docker.sh rename to _internal/templating/.internal_tei/roll_out_docker.sh diff --git a/11-embeddings-reranker-classification-tensorrt/templating/README.md b/_internal/templating/README.md similarity index 100% rename from 11-embeddings-reranker-classification-tensorrt/templating/README.md rename to _internal/templating/README.md diff --git a/11-embeddings-reranker-classification-tensorrt/templating/deploy_all.py b/_internal/templating/deploy_all.py similarity index 100% rename from 11-embeddings-reranker-classification-tensorrt/templating/deploy_all.py rename to _internal/templating/deploy_all.py diff --git a/11-embeddings-reranker-classification-tensorrt/templating/generate_templates.py b/_internal/templating/generate_templates.py similarity index 98% rename from 11-embeddings-reranker-classification-tensorrt/templating/generate_templates.py rename to _internal/templating/generate_templates.py index 05caffdbd..093c1819b 100644 --- a/11-embeddings-reranker-classification-tensorrt/templating/generate_templates.py +++ b/_internal/templating/generate_templates.py @@ -33,8 +33,16 @@ import copy REPO_URL = "https://github.com/basetenlabs/truss-examples" -SUBFOLDER = Path("11-embeddings-reranker-classification-tensorrt") ROOT_NAME = Path(REPO_URL.split("/")[-1]) + +# Map each solution type to its output subdirectory +SUBFOLDER_MAP = { + "BEI": Path("embeddings/bei"), + "BEI-Bert": Path("embeddings/bei"), + "TEI": Path("embeddings/tei"), + "Briton": Path("optimized/briton"), + "BISV2": Path("optimized/bisv2"), +} BEI_VERSION = os.environ.get("BEI") ENGINE_BUILDER_VERSION = os.environ.get("ENGINE_BUILDER") BRITON_VERSION = os.environ.get("BRITON") @@ -1150,7 +1158,20 @@ def generate_deployment(dp: Deployment): root = Path(__file__).parent.parent.parent assert root.name == ROOT_NAME.name, "This script has been moved" - folder_relative_path = SUBFOLDER / dp.folder_name + subfolder = SUBFOLDER_MAP.get(dp.solution.nickname) + if subfolder is None: + raise ValueError( + f"No SUBFOLDER_MAP entry for solution nickname '{dp.solution.nickname}'" + ) + # Strip the solution nickname prefix from the folder name since directory + # names no longer include the BEI-/TEI-/Briton-/BISV2- prefix + raw_folder = dp.folder_name + prefix = dp.solution.nickname + "-" + if raw_folder.startswith(prefix): + stripped_folder = raw_folder[len(prefix):] + else: + stripped_folder = raw_folder + folder_relative_path = subfolder / stripped_folder full_folder_path = root / folder_relative_path is_gated_notice = ( "Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). " @@ -2339,10 +2360,14 @@ def format_filter(dps: list[Deployment], type_): sorted_filter = sorted( [dp for dp in dps if isinstance(dp.task, type_)], key=lambda x: x.name ) - names = [ - f"[{dp.name}-{dp.solution.nickname}]({REPO_URL}/tree/main/{SUBFOLDER}/{dp.folder_name})" - for dp in sorted_filter - ] + names = [] + for dp in sorted_filter: + subfolder = SUBFOLDER_MAP.get(dp.solution.nickname, Path("")) + prefix = dp.solution.nickname + "-" + folder = dp.folder_name[len(prefix):] if dp.folder_name.startswith(prefix) else dp.folder_name + names.append( + f"[{dp.name}-{dp.solution.nickname}]({REPO_URL}/tree/main/{subfolder}/{folder})" + ) names_fmt = "\n - ".join(names) names_fmt = " - " + names_fmt return names_fmt @@ -2404,5 +2429,6 @@ def format_filter(dps: list[Deployment], type_): Examples: {generation_names_fmt} """ - (Path(__file__).parent.parent / "README.md").write_text(readme) + # Write generated README to _internal/ (not repo root β€” that's manually maintained) + (Path(__file__).parent / "GENERATED_README.md").write_text(readme) print(readme) diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/README.md b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/README.md similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/README.md rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/README.md diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/config.yaml b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/config.yaml similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/config.yaml rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/config.yaml diff --git a/09-private-huggingface/model/__init__.py b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/model/__init__.py similarity index 100% rename from 09-private-huggingface/model/__init__.py rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/model/__init__.py diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/README.md b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/README.md similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/README.md rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/README.md diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/config.yaml b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/config.yaml similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/config.yaml rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/config.yaml diff --git a/10-using-system-packages/model/__init__.py b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/model/__init__.py similarity index 100% rename from 10-using-system-packages/model/__init__.py rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/model/__init__.py diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/README.md b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/README.md similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/README.md rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/README.md diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/config.yaml b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/config.yaml similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/config.yaml rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/config.yaml diff --git a/audiogen-medium/model/__init__.py b/_internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/model/__init__.py similarity index 100% rename from audiogen-medium/model/__init__.py rename to _internal/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/model/__init__.py diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/README.md b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/README.md similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/README.md rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/README.md diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/config.yaml b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/config.yaml similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/config.yaml rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/config.yaml diff --git a/autodesk-wala/model/__init__.py b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/model/__init__.py similarity index 100% rename from autodesk-wala/model/__init__.py rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/model/__init__.py diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/README.md b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/README.md similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/README.md rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/README.md diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/config.yaml b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/config.yaml similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/config.yaml rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/config.yaml diff --git a/autodesk-wala/packages/src/__init__.py b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/__init__.py rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/model/__init__.py diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/README.md b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/README.md similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/README.md rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/README.md diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/config.yaml b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/config.yaml similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/config.yaml rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/config.yaml diff --git a/autodesk-wala/packages/src/diffusion_modules/__init__.py b/_internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/__init__.py rename to _internal/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/model/__init__.py diff --git a/audio/README.md b/audio/README.md new file mode 100644 index 000000000..a3af29879 --- /dev/null +++ b/audio/README.md @@ -0,0 +1,33 @@ +# Audio Models + +Truss configurations for speech-to-text, text-to-speech, music generation, and multimodal audio models. Includes both batch and streaming deployments. + +| Directory | Variants | Description | +|-----------|----------|-------------| +| [whisper](whisper/) | 8 | OpenAI Whisper speech-to-text models including Faster Whisper, WhisperX, and streaming variants | +| [kokoro](kokoro/) | 1 | Kokoro text-to-speech model | +| [chatterbox-tts](chatterbox-tts/) | 1 | Chatterbox text-to-speech model | +| [piper-tts](piper-tts/) | 1 | Piper lightweight text-to-speech engine | +| [xtts-v2](xtts-v2/) | 1 | Coqui XTTS v2 multilingual text-to-speech | +| [xtts-streaming](xtts-streaming/) | 1 | Coqui XTTS with streaming audio output | +| [orpheus-3b-websockets](orpheus-3b-websockets/) | 1 | Orpheus 3B TTS with WebSocket streaming | +| [orpheus-best-performance](orpheus-best-performance/) | 1 | Orpheus TTS optimized for lowest latency | +| [sesame-csm-1b](sesame-csm-1b/) | 1 | Sesame CSM 1B conversational speech model | +| [metavoice-1b](metavoice-1b/) | 1 | MetaVoice 1B text-to-speech model | +| [ultravox](ultravox/) | 1 | Ultravox multimodal audio-language model | +| [audiogen-medium](audiogen-medium/) | 1 | Meta AudioGen medium audio generation from text | +| [musicgen-large](musicgen-large/) | 1 | Meta MusicGen large music generation | +| [musicgen-melody](musicgen-melody/) | 1 | Meta MusicGen melody-conditioned music generation | +| [nvidia-parakeet](nvidia-parakeet/) | 1 | NVIDIA Parakeet automatic speech recognition | +| [qwen-asr](qwen-asr/) | 1 | Qwen automatic speech recognition model | +| [qwen-omni](qwen-omni/) | 1 | Qwen Omni multimodal audio model | +| [qwen-omni-thinker](qwen-omni-thinker/) | 1 | Qwen Omni Thinker reasoning audio model | +| [voxtral-streaming-4b](voxtral-streaming-4b/) | 1 | Mistral Voxtral 4B streaming speech model | + +## Deploying + +Each audio model can be deployed to Baseten with: + +```bash +truss push +``` diff --git a/audio/audiogen-medium/README.md b/audio/audiogen-medium/README.md new file mode 100644 index 000000000..ddf3be8e2 --- /dev/null +++ b/audio/audiogen-medium/README.md @@ -0,0 +1,36 @@ +# AudioGen medium + +AudioGen is a simple and controllable model for audio generation developed by Facebook AI Research. + +| Property | Value | +|----------|-------| +| Task | Audio generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "duration": 8, + "prompts": [ + "dog barking", + "sirene of an emergency vehicle", + "footsteps in a corridor" + ] +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/audiogen-medium/config.yaml b/audio/audiogen-medium/config.yaml similarity index 92% rename from audiogen-medium/config.yaml rename to audio/audiogen-medium/config.yaml index 7eaad77a3..98d8d0752 100644 --- a/audiogen-medium/config.yaml +++ b/audio/audiogen-medium/config.yaml @@ -3,6 +3,7 @@ description: AudioGen is a simple and controllable model for audio generation de environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "facebook/audiogen-medium" avatar_url: https://cdn.baseten.co/production/static/explore/meta.png cover_image_url: https://cdn.baseten.co/production/static/explore/musicgen-cover.png example_model_input: @@ -18,7 +19,7 @@ python_version: py39 requirements: - torch>=2 - git+https://github.com/facebookresearch/audiocraft.git -- torchaudio +- torchaudio==2.1.0 resources: accelerator: A10G cpu: '3' diff --git a/autodesk-wala/packages/src/experiments/__init__.py b/audio/audiogen-medium/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/experiments/__init__.py rename to audio/audiogen-medium/model/__init__.py diff --git a/audiogen-medium/model/model.py b/audio/audiogen-medium/model/model.py similarity index 100% rename from audiogen-medium/model/model.py rename to audio/audiogen-medium/model/model.py diff --git a/audio/chatterbox-tts/README.md b/audio/chatterbox-tts/README.md new file mode 100644 index 000000000..ec709e9a6 --- /dev/null +++ b/audio/chatterbox-tts/README.md @@ -0,0 +1,31 @@ +# Chatterbox TTS + +Deploy Chatterbox TTS for text-to-speech on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text-to-speech | +| Engine | Custom (Truss) | +| GPU | H100 | +| Python | py312 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"text": "Hello, this is a test of text to speech."}' +``` + +## Configuration highlights + +- Base image: `jojobaseten/truss-numpy-1.26.0-gpu:0.4` diff --git a/chatterbox-tts/config.yaml b/audio/chatterbox-tts/config.yaml similarity index 59% rename from chatterbox-tts/config.yaml rename to audio/chatterbox-tts/config.yaml index ea5f031b9..46aed1f53 100644 --- a/chatterbox-tts/config.yaml +++ b/audio/chatterbox-tts/config.yaml @@ -1,10 +1,14 @@ +description: "Chatterbox TTS for text-to-speech" +model_metadata: + repo_id: "resemble-ai/chatterbox" + example_model_input: {"text": "Hello, how are you today?"} model_name: Chatterbox TTS base_image: image: jojobaseten/truss-numpy-1.26.0-gpu:0.4 python_executable_path: /usr/bin/python3 python_version: py312 requirements: - - chatterbox-tts + - chatterbox-tts==0.1.1 resources: accelerator: H100 cpu: '1' diff --git a/chatterbox-tts/docker/Dockerfile b/audio/chatterbox-tts/docker/Dockerfile similarity index 100% rename from chatterbox-tts/docker/Dockerfile rename to audio/chatterbox-tts/docker/Dockerfile diff --git a/chatterbox-tts/docker/docker_build.sh b/audio/chatterbox-tts/docker/docker_build.sh similarity index 100% rename from chatterbox-tts/docker/docker_build.sh rename to audio/chatterbox-tts/docker/docker_build.sh diff --git a/chatterbox-tts/input/obama_8s.wav b/audio/chatterbox-tts/input/obama_8s.wav similarity index 100% rename from chatterbox-tts/input/obama_8s.wav rename to audio/chatterbox-tts/input/obama_8s.wav diff --git a/autodesk-wala/packages/src/experiments/utils/__init__.py b/audio/chatterbox-tts/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/experiments/utils/__init__.py rename to audio/chatterbox-tts/model/__init__.py diff --git a/chatterbox-tts/model/model.py b/audio/chatterbox-tts/model/model.py similarity index 100% rename from chatterbox-tts/model/model.py rename to audio/chatterbox-tts/model/model.py diff --git a/chatterbox-tts/run_tts.py b/audio/chatterbox-tts/run_tts.py similarity index 100% rename from chatterbox-tts/run_tts.py rename to audio/chatterbox-tts/run_tts.py diff --git a/audio/kokoro/README.md b/audio/kokoro/README.md new file mode 100644 index 000000000..661a3e716 --- /dev/null +++ b/audio/kokoro/README.md @@ -0,0 +1,34 @@ +# kokoro + +Deploy kokoro for text-to-speech on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text-to-speech | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Kokoro is a frontier TTS model for its size of 82 million parameters (text in/audio out). On 25 Dec 2024, Kokoro v0.19 weights were permissively released in full fp32 precision under an Apache 2.0 license. As of 2 Jan 2025, 10 unique Voicepacks have been released, and a .onnx version of v0.19 is available.In the weeks leading up to its release, Kokoro v0.19 was the #1\ud83e\udd47 ranked model in TTS Spaces Arena. Kokoro had achieved higher Elo in this single-voice Arena setting over other models, using fewer parameters and less data. Kokoro's ability to top this Elo ladder suggests that the scaling law (Elo vs compute/data/params) for traditional TTS models might have a steeper slope than previously expected.", + "voice": "af", + "speed": 1.0 +}' +``` + +## Configuration highlights + +- Predict concurrency: **1** +- System packages: `espeak-ng` diff --git a/kokoro/call.py b/audio/kokoro/call.py similarity index 100% rename from kokoro/call.py rename to audio/kokoro/call.py diff --git a/kokoro/config.yaml b/audio/kokoro/config.yaml similarity index 90% rename from kokoro/config.yaml rename to audio/kokoro/config.yaml index 3d645e206..68f8cc06f 100644 --- a/kokoro/config.yaml +++ b/audio/kokoro/config.yaml @@ -1,7 +1,9 @@ +description: "kokoro for text-to-speech" build_commands: - python3 -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab')" environment_variables: {} model_metadata: + repo_id: "hexgrad/Kokoro-82M" example_model_input: {"text": "Kokoro is a frontier TTS model for its size of 82 million parameters (text in/audio out). On 25 Dec 2024, Kokoro v0.19 weights were permissively released in full fp32 precision under an Apache 2.0 license. As of 2 Jan 2025, 10 unique Voicepacks have been released, and a .onnx version of v0.19 is available.In the weeks leading up to its release, Kokoro v0.19 was the #1πŸ₯‡ ranked model in TTS Spaces Arena. Kokoro had achieved higher Elo in this single-voice Arena setting over other models, using fewer parameters and less data. Kokoro's ability to top this Elo ladder suggests that the scaling law (Elo vs compute/data/params) for traditional TTS models might have a steeper slope than previously expected.", "voice": "af", "speed": 1.0} model_name: kokoro python_version: py311 @@ -11,8 +13,8 @@ requirements: - scipy==1.15.1 - phonemizer==3.3.0 - nltk==3.9.1 -- numpy -- huggingface_hub[hf_transfer] +- numpy==1.26.0 +- huggingface_hub[hf_transfer]==0.19.4 - hf_transfer==0.1.9 - munch==4.0.0 resources: diff --git a/autodesk-wala/packages/src/latent_model/__init__.py b/audio/kokoro/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/__init__.py rename to audio/kokoro/model/__init__.py diff --git a/kokoro/model/model.py b/audio/kokoro/model/model.py similarity index 100% rename from kokoro/model/model.py rename to audio/kokoro/model/model.py diff --git a/audio/metavoice-1b/README.md b/audio/metavoice-1b/README.md new file mode 100644 index 000000000..a516a7a9a --- /dev/null +++ b/audio/metavoice-1b/README.md @@ -0,0 +1,32 @@ +# MetaVoice 1B + +MetaVoice is a transformer-based model for TTS + +| Property | Value | +|----------|-------| +| Model | [metavoiceio/metavoice-1B-v0.1](https://huggingface.co/metavoiceio/metavoice-1B-v0.1) | +| Task | Text-to-speech | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '"text to speech models are cool"' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/metavoice-1b/config.yaml b/audio/metavoice-1b/config.yaml similarity index 94% rename from metavoice-1b/config.yaml rename to audio/metavoice-1b/config.yaml index e37e4c6ec..191223b0a 100644 --- a/metavoice-1b/config.yaml +++ b/audio/metavoice-1b/config.yaml @@ -12,9 +12,11 @@ model_cache: allow_patterns: - "*.pt" - repo_id: facebook/multiband-diffusion + use_volume: false allow_patterns: - mbd_comp_8.pt - repo_id: facebook/encodec_24khz + use_volume: false allow_patterns: - "*.safetensors" diff --git a/metavoice-1b/data/bria.mp3 b/audio/metavoice-1b/data/bria.mp3 similarity index 100% rename from metavoice-1b/data/bria.mp3 rename to audio/metavoice-1b/data/bria.mp3 diff --git a/autodesk-wala/packages/src/mvdream/ldm/__init__.py b/audio/metavoice-1b/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/__init__.py rename to audio/metavoice-1b/model/__init__.py diff --git a/metavoice-1b/model/model.py b/audio/metavoice-1b/model/model.py similarity index 100% rename from metavoice-1b/model/model.py rename to audio/metavoice-1b/model/model.py diff --git a/metavoice-1b/process.py b/audio/metavoice-1b/process.py similarity index 100% rename from metavoice-1b/process.py rename to audio/metavoice-1b/process.py diff --git a/metavoice-1b/requirements.txt b/audio/metavoice-1b/requirements.txt similarity index 100% rename from metavoice-1b/requirements.txt rename to audio/metavoice-1b/requirements.txt diff --git a/audio/musicgen-large/README.md b/audio/musicgen-large/README.md new file mode 100644 index 000000000..604590fbd --- /dev/null +++ b/audio/musicgen-large/README.md @@ -0,0 +1,36 @@ +# MusicGen large + +MusicGen is a simple and controllable model for music generation developed by Facebook AI Research. + +| Property | Value | +|----------|-------| +| Task | Audio generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "duration": 8, + "prompts": [ + "happy rock", + "energetic EDM", + "sad jazz" + ] +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/musicgen-large/config.yaml b/audio/musicgen-large/config.yaml similarity index 92% rename from musicgen-large/config.yaml rename to audio/musicgen-large/config.yaml index 3f6a7e9f7..1475c042d 100644 --- a/musicgen-large/config.yaml +++ b/audio/musicgen-large/config.yaml @@ -3,6 +3,7 @@ description: MusicGen is a simple and controllable model for music generation de environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "facebook/musicgen-large" avatar_url: https://cdn.baseten.co/production/static/explore/meta.png cover_image_url: https://cdn.baseten.co/production/static/explore/musicgen-cover.png example_model_input: @@ -17,7 +18,7 @@ model_name: MusicGen large python_version: py39 requirements: - torch>=2 -- audiocraft +- audiocraft==1.3.0 resources: accelerator: A10G cpu: '3' diff --git a/autodesk-wala/packages/src/mvdream/ldm/models/__init__.py b/audio/musicgen-large/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/models/__init__.py rename to audio/musicgen-large/model/__init__.py diff --git a/musicgen-large/model/model.py b/audio/musicgen-large/model/model.py similarity index 100% rename from musicgen-large/model/model.py rename to audio/musicgen-large/model/model.py diff --git a/audio/musicgen-melody/README.md b/audio/musicgen-melody/README.md new file mode 100644 index 000000000..32afc8be1 --- /dev/null +++ b/audio/musicgen-melody/README.md @@ -0,0 +1,36 @@ +# MusicGen Melody + +MusicGen Melody is a simple and controllable model for music generation conditioned on text and audio. It is developed by Facebook AI Research. + +| Property | Value | +|----------|-------| +| Task | Audio generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "duration": 8, + "prompts": [ + "happy rock", + "energetic EDM", + "sad jazz" + ] +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/musicgen-melody/config.yaml b/audio/musicgen-melody/config.yaml similarity index 90% rename from musicgen-melody/config.yaml rename to audio/musicgen-melody/config.yaml index ad71431a4..82fe49b63 100644 --- a/musicgen-melody/config.yaml +++ b/audio/musicgen-melody/config.yaml @@ -3,6 +3,7 @@ description: MusicGen Melody is a simple and controllable model for music genera environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "facebook/musicgen-melody" avatar_url: https://cdn.baseten.co/production/static/explore/meta.png cover_image_url: https://cdn.baseten.co/production/static/explore/musicgen-cover.png example_model_input: @@ -17,8 +18,8 @@ model_name: MusicGen Melody python_version: py39 requirements: - torch>=2 -- audiocraft -- protobuf +- audiocraft==1.3.0 +- protobuf==4.25.1 resources: accelerator: A10G cpu: '3' diff --git a/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/__init__.py b/audio/musicgen-melody/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/models/diffusion/__init__.py rename to audio/musicgen-melody/model/__init__.py diff --git a/musicgen-melody/model/model.py b/audio/musicgen-melody/model/model.py similarity index 100% rename from musicgen-melody/model/model.py rename to audio/musicgen-melody/model/model.py diff --git a/audio/nvidia-parakeet/README.md b/audio/nvidia-parakeet/README.md new file mode 100644 index 000000000..26b6b7f87 --- /dev/null +++ b/audio/nvidia-parakeet/README.md @@ -0,0 +1,36 @@ +# Parakeet TDT 0.6B V2 + +Parakeet TDT 0.6B V2 is a 600-million-parameter automatic speech recognition (ASR) model designed for high-quality English transcription. + +| Property | Value | +|----------|-------| +| Model | [nvidia/parakeet-tdt-0.6b-v2](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2) | +| Task | Audio | +| Engine | Custom (Truss) | +| GPU | H100_40GB | +| Python | py312 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "audio_url": "https://dldata-public.s3.us-east-2.amazonaws.com/2086-149220-0033.wav", + "timestamps": false +}' +``` + +## Configuration highlights + +- Predict concurrency: **8** +- System packages: `ffmpeg` diff --git a/nvidia/parakeet-tdt-0_6b-v2/config.yaml b/audio/nvidia-parakeet/config.yaml similarity index 93% rename from nvidia/parakeet-tdt-0_6b-v2/config.yaml rename to audio/nvidia-parakeet/config.yaml index a8ffc2aa8..ae599d138 100644 --- a/nvidia/parakeet-tdt-0_6b-v2/config.yaml +++ b/audio/nvidia-parakeet/config.yaml @@ -19,7 +19,7 @@ model_name: Parakeet TDT 0.6B V2 secrets: hf_access_token: null requirements: - - nemo_toolkit[asr] - - requests + - nemo_toolkit[asr]==2.2.0 + - requests==2.32.3 - pyarrow==20.0.0 - cuda-python>=12.3 diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/__init__.py b/audio/nvidia-parakeet/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/__init__.py rename to audio/nvidia-parakeet/model/__init__.py diff --git a/nvidia/parakeet-tdt-0_6b-v2/model/model.py b/audio/nvidia-parakeet/model/model.py similarity index 100% rename from nvidia/parakeet-tdt-0_6b-v2/model/model.py rename to audio/nvidia-parakeet/model/model.py diff --git a/audio/orpheus-3b-websockets/README.md b/audio/orpheus-3b-websockets/README.md new file mode 100644 index 000000000..40d213a9e --- /dev/null +++ b/audio/orpheus-3b-websockets/README.md @@ -0,0 +1,40 @@ +# Orpheus-3b Websockets + +Deploy Orpheus-3b Websockets on Baseten using a TRT-LLM engine. + +| Property | Value | +|----------|-------| +| Model | [baseten/orpheus-3b-0.1-ft](https://huggingface.co/baseten/orpheus-3b-0.1-ft) | +| Task | Audio | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 KV | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "max_tokens": 10000, + "prompt": "In todays fast-paced world, finding balance between work and personal life is more important than ever. With the constant demands of technology, remote communication, ", + "voice": "tara" +}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **65,536** +- Plugin: **use_fp8_context_fmha** +- Environment variables: `ENABLE_EXECUTOR_API` diff --git a/orpheus-3b-websockets/call.py b/audio/orpheus-3b-websockets/call.py similarity index 100% rename from orpheus-3b-websockets/call.py rename to audio/orpheus-3b-websockets/call.py diff --git a/orpheus-3b-websockets/config.yaml b/audio/orpheus-3b-websockets/config.yaml similarity index 93% rename from orpheus-3b-websockets/config.yaml rename to audio/orpheus-3b-websockets/config.yaml index eae3cae0b..587f32b30 100644 --- a/orpheus-3b-websockets/config.yaml +++ b/audio/orpheus-3b-websockets/config.yaml @@ -1,3 +1,4 @@ +description: "Orpheus-3b Websockets for text-to-speech" build_commands: - apt-get update && apt-get install git git-lfs -y - git lfs install @@ -20,9 +21,9 @@ requirements: - snac==1.2.1 - torch==2.7.0 - batched==0.1.4 - - httpx - - websockets - - pysbd + - httpx==0.27.0 + - websockets==13.1 + - pysbd==0.3.4 resources: accelerator: H100_40GB cpu: "1" diff --git a/orpheus-3b-websockets/model/model.py b/audio/orpheus-3b-websockets/model/model.py similarity index 100% rename from orpheus-3b-websockets/model/model.py rename to audio/orpheus-3b-websockets/model/model.py diff --git a/orpheus-3b-websockets/snac_batching_quantization_dev.py b/audio/orpheus-3b-websockets/snac_batching_quantization_dev.py similarity index 100% rename from orpheus-3b-websockets/snac_batching_quantization_dev.py rename to audio/orpheus-3b-websockets/snac_batching_quantization_dev.py diff --git a/audio/orpheus-best-performance/README.md b/audio/orpheus-best-performance/README.md new file mode 100644 index 000000000..d6822e71e --- /dev/null +++ b/audio/orpheus-best-performance/README.md @@ -0,0 +1,42 @@ +# Orpheus-3b Best Performance + +Deploy Orpheus-3b Best Performance on Baseten using a TRT-LLM engine. + +| Property | Value | +|----------|-------| +| Model | [canopylabs/orpheus-3b-0.1-ft](https://huggingface.co/canopylabs/orpheus-3b-0.1-ft) | +| Task | Audio | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 KV | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "max_tokens": 10000, + "prompt": "In todays fast-paced world, finding balance between work and personal life is more important than ever. With the constant demands of technology, remote communication, ", + "voice": "tara" +}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_fp8_context_fmha** +- Environment variables: `ENABLE_EXECUTOR_API` diff --git a/orpheus-best-performance/call.py b/audio/orpheus-best-performance/call.py similarity index 100% rename from orpheus-best-performance/call.py rename to audio/orpheus-best-performance/call.py diff --git a/orpheus-best-performance/config.yaml b/audio/orpheus-best-performance/config.yaml similarity index 96% rename from orpheus-best-performance/config.yaml rename to audio/orpheus-best-performance/config.yaml index 322d77e10..e73e9f4d4 100644 --- a/orpheus-best-performance/config.yaml +++ b/audio/orpheus-best-performance/config.yaml @@ -1,3 +1,4 @@ +description: "Orpheus-3b Best Performance for text-to-speech" build_commands: - apt-get update && apt-get install git git-lfs -y - git lfs install diff --git a/orpheus-best-performance/model/model.py b/audio/orpheus-best-performance/model/model.py similarity index 100% rename from orpheus-best-performance/model/model.py rename to audio/orpheus-best-performance/model/model.py diff --git a/orpheus-best-performance/snac_batching_quantization_dev.py b/audio/orpheus-best-performance/snac_batching_quantization_dev.py similarity index 100% rename from orpheus-best-performance/snac_batching_quantization_dev.py rename to audio/orpheus-best-performance/snac_batching_quantization_dev.py diff --git a/audio/piper-tts/README.md b/audio/piper-tts/README.md new file mode 100644 index 000000000..843cbeaa8 --- /dev/null +++ b/audio/piper-tts/README.md @@ -0,0 +1,31 @@ +# Piper TTS + +Deploy Piper TTS for text-to-speech on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text-to-speech | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "text": "I love robots. Robots are cool!" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/piper-tts/config.yaml b/audio/piper-tts/config.yaml similarity index 89% rename from piper-tts/config.yaml rename to audio/piper-tts/config.yaml index 63b890b52..fad82ed4a 100644 --- a/piper-tts/config.yaml +++ b/audio/piper-tts/config.yaml @@ -1,3 +1,4 @@ +description: "Piper TTS for text-to-speech" environment_variables: {} external_data: - local_data_path: models/model.onnx @@ -6,6 +7,7 @@ external_data: url: https://huggingface.co/rhasspy/piper-voices/raw/v1.0.0/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json external_package_dirs: [] model_metadata: + repo_id: "rhasspy/piper-voices" example_model_input: text: I love robots. Robots are cool! tags: diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/__init__.py b/audio/piper-tts/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/__init__.py rename to audio/piper-tts/model/__init__.py diff --git a/piper-tts/model/model.py b/audio/piper-tts/model/model.py similarity index 100% rename from piper-tts/model/model.py rename to audio/piper-tts/model/model.py diff --git a/audio/qwen-asr/README.md b/audio/qwen-asr/README.md new file mode 100644 index 000000000..e25eb7ea9 --- /dev/null +++ b/audio/qwen-asr/README.md @@ -0,0 +1,49 @@ +# Qwen3-ASR-1.7B + +Deploy Qwen3-ASR-1.7B on Baseten using a vLLM engine. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) | +| Task | Audio | +| Engine | vLLM | +| GPU | H100_40GB:1 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "stream": false, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "audio_url", + "audio_url": { + "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav" + } + } + ] + } + ] +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:nightly-070c811d6f74c55302557878f5982411a3346b4d` +- Predict concurrency: **256** +- System packages: `python3.10-venv, ffmpeg, openmpi-bin, libopenmpi-dev` diff --git a/qwen/qwen-3-asr/config.yaml b/audio/qwen-asr/config.yaml similarity index 84% rename from qwen/qwen-3-asr/config.yaml rename to audio/qwen-asr/config.yaml index a86c79a4f..cf55fe465 100644 --- a/qwen/qwen-3-asr/config.yaml +++ b/audio/qwen-asr/config.yaml @@ -1,4 +1,6 @@ +description: "Qwen3-ASR-1.7B for speech-to-text" model_metadata: + repo_id: "Qwen/Qwen3-ASR-1.7B" example_model_input: stream: false messages: @@ -28,11 +30,11 @@ resources: requirements: - --pre --extra-index-url https://wheels.vllm.ai/nightly - vllm[audio] - - librosa - - torch - - torchaudio - - pynvml - - ffmpeg-python + - librosa==0.10.2 + - torch==2.5.1 + - torchaudio==2.5.1 + - pynvml==11.5.3 + - ffmpeg-python==0.2.0 system_packages: - python3.10-venv - ffmpeg diff --git a/audio/qwen-omni-thinker/README.md b/audio/qwen-omni-thinker/README.md new file mode 100644 index 000000000..91c0a94bf --- /dev/null +++ b/audio/qwen-omni-thinker/README.md @@ -0,0 +1,67 @@ +# Qwen3 Omni 30B Instruct (Thinker Only) + +Deploy Qwen3 Omni 30B Instruct (Thinker Only) on Baseten using a vLLM engine. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-Omni-30B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct) | +| Task | Audio | +| Engine | vLLM | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe this image and audio content." + }, + { + "type": "image_url", + "image_url": { + "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg" + } + }, + { + "type": "audio_url", + "audio_url": { + "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cough.wav" + } + }, + { + "type": "text", + "text": "What can you see and hear? Answer in one sentence." + } + ] + } + ], + "stream": false, + "model": "qwen3-omni", + "max_tokens": 2048, + "temperature": 0.7 +}' +``` + +## Configuration highlights + +- Base image: `qwenllm/qwen3-omni:3-cu124` +- Predict concurrency: **32** diff --git a/qwen/qwen-3-30b-omni-thinker/config.yaml b/audio/qwen-omni-thinker/config.yaml similarity index 90% rename from qwen/qwen-3-30b-omni-thinker/config.yaml rename to audio/qwen-omni-thinker/config.yaml index 81cc44e0d..4f740a03a 100644 --- a/qwen/qwen-3-30b-omni-thinker/config.yaml +++ b/audio/qwen-omni-thinker/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen3 Omni 30B Instruct (Thinker Only) for audio understanding" model_name: Qwen3 Omni 30B Instruct (Thinker Only) base_image: image: qwenllm/qwen3-omni:3-cu124 @@ -9,6 +10,7 @@ docker_server: predict_endpoint: /v1/chat/completions server_port: 8000 model_metadata: + repo_id: "Qwen/Qwen3-Omni-30B-A3B-Instruct" example_model_input: messages: - role: system diff --git a/audio/qwen-omni/README.md b/audio/qwen-omni/README.md new file mode 100644 index 000000000..5c6be2571 --- /dev/null +++ b/audio/qwen-omni/README.md @@ -0,0 +1,43 @@ +# Qwen3 Omni 30B Instruct + +Deploy Qwen3 Omni 30B Instruct on Baseten using a Custom (Truss) engine. + +| Property | Value | +|----------|-------| +| Task | Audio | +| Engine | Custom (Truss) | +| GPU | H100 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "speaker": "Chelsie", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hi, how are you?" + } + ] + } + ] +}' +``` + +## Configuration highlights + +- Base image: `qwenllm/qwen3-omni` +- Predict concurrency: **1** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/qwen/qwen-3-30b-omni/config.yaml b/audio/qwen-omni/config.yaml similarity index 82% rename from qwen/qwen-3-30b-omni/config.yaml rename to audio/qwen-omni/config.yaml index af7b83393..94d2638db 100644 --- a/qwen/qwen-3-30b-omni/config.yaml +++ b/audio/qwen-omni/config.yaml @@ -1,7 +1,9 @@ +description: "Qwen3 Omni 30B Instruct for audio understanding" model_name: Qwen3 Omni 30B Instruct base_image: image: qwenllm/qwen3-omni model_metadata: + repo_id: "Qwen/Qwen3-Omni-30B-A3B-Instruct" example_model_input: { "speaker": "Chelsie", "messages": [ diff --git a/qwen/qwen-3-30b-omni/model/model.py b/audio/qwen-omni/model/model.py similarity index 100% rename from qwen/qwen-3-30b-omni/model/model.py rename to audio/qwen-omni/model/model.py diff --git a/audio/sesame-csm-1b/README.md b/audio/sesame-csm-1b/README.md new file mode 100644 index 000000000..29190f6ce --- /dev/null +++ b/audio/sesame-csm-1b/README.md @@ -0,0 +1,34 @@ +# sesame-csm-1b + +Deploy sesame-csm-1b for text-to-speech on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text-to-speech | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Hello from Sesame.", + "speaker": 0 +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/sesame-csm-1b/config.yaml b/audio/sesame-csm-1b/config.yaml similarity index 71% rename from sesame-csm-1b/config.yaml rename to audio/sesame-csm-1b/config.yaml index be19e9d10..e38f325fb 100644 --- a/sesame-csm-1b/config.yaml +++ b/audio/sesame-csm-1b/config.yaml @@ -1,6 +1,8 @@ +description: "sesame-csm-1b for text-to-speech" model_name: sesame-csm-1b python_version: py310 model_metadata: + repo_id: "sesame/csm-1b" example_model_input: text: "Hello from Sesame." speaker: 0 @@ -13,9 +15,9 @@ requirements: - moshi==0.2.2 - torchtune==0.4.0 - torchao==0.9.0 - - silentcipher @ git+https://github.com/SesameAILabs/silentcipher@master - - ffmpeg - - git+https://github.com/veerbia/csm.git + - silentcipher @ git+https://github.com/SesameAILabs/silentcipher@d46d7d0893a583d8968ab3a6626e2289faec9152 + - ffmpeg==1.4 + - git+https://github.com/veerbia/csm.git@f747b9652b1c3859ce0f0465176ef8dff4cb32c8 resources: accelerator: T4 cpu: '1' diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/distributions/__init__.py b/audio/sesame-csm-1b/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/distributions/__init__.py rename to audio/sesame-csm-1b/model/__init__.py diff --git a/sesame-csm-1b/model/model.py b/audio/sesame-csm-1b/model/model.py similarity index 100% rename from sesame-csm-1b/model/model.py rename to audio/sesame-csm-1b/model/model.py diff --git a/audio/ultravox/README.md b/audio/ultravox/README.md new file mode 100644 index 000000000..0fb9e631d --- /dev/null +++ b/audio/ultravox/README.md @@ -0,0 +1,31 @@ +# Ultravox v0.2 + +Deploy Ultravox v0.2 for audio understanding on Baseten. + +| Property | Value | +|----------|-------| +| Task | Audio understanding | +| Engine | vLLM | +| GPU | A100 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "fixie-ai/ultravox-v0.2", "messages": [{"role": "user", "content": "Describe this audio."}]}' +``` + +## Configuration highlights + +- Base image: `vshulman/vllm-openai-fixie:latest` +- Predict concurrency: **512** +- System packages: `python3.10-venv` diff --git a/ultravox/config.yaml b/audio/ultravox/config.yaml similarity index 61% rename from ultravox/config.yaml rename to audio/ultravox/config.yaml index b56bf0c5c..b1b05a8db 100644 --- a/ultravox/config.yaml +++ b/audio/ultravox/config.yaml @@ -1,7 +1,10 @@ +description: "Ultravox v0.2 for audio understanding" base_image: image: vshulman/vllm-openai-fixie:latest python_executable_path: /usr/bin/python3 model_metadata: + repo_id: "fixie-ai/ultravox-v0_4_1-llama-3_1-8b" + example_model_input: {"messages": [{"role": "user", "content": "Describe the audio clip."}], "model": "fixie-ai/ultravox-v0.2", "max_tokens": 512} arguments: model: fixie-ai/ultravox-v0.2 audio_token_id: 128002 @@ -12,7 +15,7 @@ python_version: py310 runtime: predict_concurrency: 512 requirements: - - httpx + - httpx==0.27.0 resources: accelerator: A100 use_gpu: true diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/encoders/__init__.py b/audio/ultravox/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/encoders/__init__.py rename to audio/ultravox/model/__init__.py diff --git a/ultravox/model/model.py b/audio/ultravox/model/model.py similarity index 100% rename from ultravox/model/model.py rename to audio/ultravox/model/model.py diff --git a/audio/voxtral-streaming-4b/README.md b/audio/voxtral-streaming-4b/README.md new file mode 100644 index 000000000..89084fd99 --- /dev/null +++ b/audio/voxtral-streaming-4b/README.md @@ -0,0 +1,46 @@ +# Voxtral-Mini-4B-Realtime-2602 + +Deploy Voxtral-Mini-4B-Realtime-2602 on Baseten using a vLLM engine. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Voxtral-Mini-4B-Realtime-2602](https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602) | +| Task | Audio | +| Engine | vLLM | +| GPU | H100_40GB:1 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model uses a WebSocket endpoint for realtime audio streaming. Use the included `streaming.py` client: + +```sh +python streaming.py +``` + +Or connect directly via WebSocket: + +```python +import websockets + +async with websockets.connect( + "wss://model-.api.baseten.co/environments/production/websocket", + extra_headers={"Authorization": "Api-Key YOUR_BASETEN_API_KEY"} +) as ws: + await ws.send('{"type": "session.update", "model": "mistralai/Voxtral-Mini-4B-Realtime-2602"}') + await ws.send('{"type": "input_audio_buffer.append", "audio": ""}') + await ws.send('{"type": "input_audio_buffer.commit"}') +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:nightly-d88a1df699f68e5284fe3a3170f8ae292a3e9c3f` +- System packages: `python3.10-venv, ffmpeg, openmpi-bin, libopenmpi-dev` +- Environment variables: `VLLM_DISABLE_COMPILE_CACHE` diff --git a/mistral/voxtral-streaming-4b/config.yaml b/audio/voxtral-streaming-4b/config.yaml similarity index 69% rename from mistral/voxtral-streaming-4b/config.yaml rename to audio/voxtral-streaming-4b/config.yaml index f7afea296..b38af3abf 100644 --- a/mistral/voxtral-streaming-4b/config.yaml +++ b/audio/voxtral-streaming-4b/config.yaml @@ -1,3 +1,7 @@ +description: "Voxtral Mini 4B for real-time speech interaction" +model_metadata: + repo_id: "mistralai/Voxtral-Mini-4B-Realtime-2602" + example_model_input: {"note": "This model uses a WebSocket endpoint at /v1/realtime. Connect via WebSocket and send audio frames for real-time speech interaction."} model_name: Voxtral-Mini-4B-Realtime-2602 secrets: hf_access_token: null @@ -19,12 +23,12 @@ resources: requirements: - --pre --extra-index-url https://wheels.vllm.ai/nightly - vllm[audio] - - librosa - - torch - - torchaudio - - pynvml - - ffmpeg-python - - websockets + - librosa==0.10.2 + - torch==2.5.1 + - torchaudio==2.5.1 + - pynvml==11.5.3 + - ffmpeg-python==0.2.0 + - websockets==13.1 system_packages: - python3.10-venv - ffmpeg diff --git a/mistral/voxtral-streaming-4b/streaming.py b/audio/voxtral-streaming-4b/streaming.py similarity index 100% rename from mistral/voxtral-streaming-4b/streaming.py rename to audio/voxtral-streaming-4b/streaming.py diff --git a/audio/whisper/faster-whisper-small/README.md b/audio/whisper/faster-whisper-small/README.md new file mode 100644 index 000000000..480cae3e9 --- /dev/null +++ b/audio/whisper/faster-whisper-small/README.md @@ -0,0 +1,32 @@ +# Faster Whisper Small + +A small speech-to-text model for multi-lingual audio transcription. + +| Property | Value | +|----------|-------| +| Model | [Systran/faster-whisper-small](https://huggingface.co/Systran/faster-whisper-small) | +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/whisper/faster-whisper-small/config.yaml b/audio/whisper/faster-whisper-small/config.yaml similarity index 100% rename from whisper/faster-whisper-small/config.yaml rename to audio/whisper/faster-whisper-small/config.yaml diff --git a/autodesk-wala/packages/src/utils/__init__.py b/audio/whisper/faster-whisper-small/model/__init__.py similarity index 100% rename from autodesk-wala/packages/src/utils/__init__.py rename to audio/whisper/faster-whisper-small/model/__init__.py diff --git a/whisper/faster-whisper-small/model/model.py b/audio/whisper/faster-whisper-small/model/model.py similarity index 100% rename from whisper/faster-whisper-small/model/model.py rename to audio/whisper/faster-whisper-small/model/model.py diff --git a/audio/whisper/faster-whisper-v2/README.md b/audio/whisper/faster-whisper-v2/README.md new file mode 100644 index 000000000..a1b7ed1e3 --- /dev/null +++ b/audio/whisper/faster-whisper-v2/README.md @@ -0,0 +1,32 @@ +# Faster Whisper v2 + +Faster Whisper v2 + +| Property | Value | +|----------|-------| +| Model | [Systran/faster-whisper-large-v2](https://huggingface.co/Systran/faster-whisper-large-v2) | +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/whisper/faster-whisper-v2/config.yaml b/audio/whisper/faster-whisper-v2/config.yaml similarity index 100% rename from whisper/faster-whisper-v2/config.yaml rename to audio/whisper/faster-whisper-v2/config.yaml diff --git a/binocular/model/__init__.py b/audio/whisper/faster-whisper-v2/model/__init__.py similarity index 100% rename from binocular/model/__init__.py rename to audio/whisper/faster-whisper-v2/model/__init__.py diff --git a/whisper/faster-whisper-v2/model/model.py b/audio/whisper/faster-whisper-v2/model/model.py similarity index 100% rename from whisper/faster-whisper-v2/model/model.py rename to audio/whisper/faster-whisper-v2/model/model.py diff --git a/audio/whisper/faster-whisper-v3/README.md b/audio/whisper/faster-whisper-v3/README.md new file mode 100644 index 000000000..a722a06de --- /dev/null +++ b/audio/whisper/faster-whisper-v3/README.md @@ -0,0 +1,32 @@ +# Faster Whisper v3 + +Faster Whisper v3 + +| Property | Value | +|----------|-------| +| Model | [Systran/faster-whisper-large-v3](https://huggingface.co/Systran/faster-whisper-large-v3) | +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/whisper/faster-whisper-v3/config.yaml b/audio/whisper/faster-whisper-v3/config.yaml similarity index 100% rename from whisper/faster-whisper-v3/config.yaml rename to audio/whisper/faster-whisper-v3/config.yaml diff --git a/chatterbox-tts/model/__init__.py b/audio/whisper/faster-whisper-v3/model/__init__.py similarity index 100% rename from chatterbox-tts/model/__init__.py rename to audio/whisper/faster-whisper-v3/model/__init__.py diff --git a/whisper/faster-whisper-v3/model/model.py b/audio/whisper/faster-whisper-v3/model/model.py similarity index 100% rename from whisper/faster-whisper-v3/model/model.py rename to audio/whisper/faster-whisper-v3/model/model.py diff --git a/gfp-gan/LICENSE b/audio/whisper/whisper-openai/LICENSE similarity index 100% rename from gfp-gan/LICENSE rename to audio/whisper/whisper-openai/LICENSE diff --git a/audio/whisper/whisper-openai/README.md b/audio/whisper/whisper-openai/README.md new file mode 100644 index 000000000..121dd66cc --- /dev/null +++ b/audio/whisper/whisper-openai/README.md @@ -0,0 +1,31 @@ +# Whisper + +Transcribe audio files across multiple languages. + +| Property | Value | +|----------|-------| +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/whisper/whisper-truss/config.yaml b/audio/whisper/whisper-openai/config.yaml similarity index 95% rename from whisper/whisper-truss/config.yaml rename to audio/whisper/whisper-openai/config.yaml index a42e4ae6d..a43e837a5 100644 --- a/whisper/whisper-truss/config.yaml +++ b/audio/whisper/whisper-openai/config.yaml @@ -5,6 +5,7 @@ external_data: url: https://baseten-public.s3.us-west-2.amazonaws.com/models/whisper/small.pt external_package_dirs: [] model_metadata: + repo_id: "openai/whisper-small" avatar_url: https://cdn.baseten.co/production/static/openai.png cover_image_url: https://cdn.baseten.co/production/static/whisper.png example_model_input: diff --git a/gfp-gan/data/.gitkeep b/audio/whisper/whisper-openai/data/.gitkeep similarity index 100% rename from gfp-gan/data/.gitkeep rename to audio/whisper/whisper-openai/data/.gitkeep diff --git a/clip/model/__init__.py b/audio/whisper/whisper-openai/model/__init__.py similarity index 100% rename from clip/model/__init__.py rename to audio/whisper/whisper-openai/model/__init__.py diff --git a/whisper/whisper-truss/model/model.py b/audio/whisper/whisper-openai/model/model.py similarity index 100% rename from whisper/whisper-truss/model/model.py rename to audio/whisper/whisper-openai/model/model.py diff --git a/audio/whisper/whisper-streaming/README.md b/audio/whisper/whisper-streaming/README.md new file mode 100644 index 000000000..71ff006f1 --- /dev/null +++ b/audio/whisper/whisper-streaming/README.md @@ -0,0 +1,30 @@ +# Whisper Streaming + +Deploy Whisper Streaming for speech-to-text transcription on Baseten. + +| Property | Value | +|----------|-------| +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"url": "https://example.com/audio.wav"}' +``` + +## Configuration highlights + +- Base image: `baseten/truss-server-base:3.10-gpu-v0.4.9` +- System packages: `ffmpeg` diff --git a/whisper/whisper-streaming/config.yaml b/audio/whisper/whisper-streaming/config.yaml similarity index 67% rename from whisper/whisper-streaming/config.yaml rename to audio/whisper/whisper-streaming/config.yaml index 9ebae7e33..0f55b7554 100644 --- a/whisper/whisper-streaming/config.yaml +++ b/audio/whisper/whisper-streaming/config.yaml @@ -1,9 +1,12 @@ +description: "Whisper Streaming for speech-to-text transcription" base_image: image: baseten/truss-server-base:3.10-gpu-v0.4.9 python_executable_path: /usr/bin/python3 environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "openai/whisper-medium" + example_model_input: {"audio": "", "chunk_size": 1.0} whisper_model: medium model_name: Whisper Streaming python_version: py310 diff --git a/cogvlm/model/__init__.py b/audio/whisper/whisper-streaming/model/__init__.py similarity index 100% rename from cogvlm/model/__init__.py rename to audio/whisper/whisper-streaming/model/__init__.py diff --git a/whisper/whisper-streaming/model/model.py b/audio/whisper/whisper-streaming/model/model.py similarity index 100% rename from whisper/whisper-streaming/model/model.py rename to audio/whisper/whisper-streaming/model/model.py diff --git a/whisper/whisper-streaming/packages/whisper_streaming/line_packet.py b/audio/whisper/whisper-streaming/packages/whisper_streaming/line_packet.py similarity index 100% rename from whisper/whisper-streaming/packages/whisper_streaming/line_packet.py rename to audio/whisper/whisper-streaming/packages/whisper_streaming/line_packet.py diff --git a/whisper/whisper-streaming/packages/whisper_streaming/whisper_online.py b/audio/whisper/whisper-streaming/packages/whisper_streaming/whisper_online.py similarity index 100% rename from whisper/whisper-streaming/packages/whisper_streaming/whisper_online.py rename to audio/whisper/whisper-streaming/packages/whisper_streaming/whisper_online.py diff --git a/whisper/whisper-streaming/packages/whisper_streaming/whisper_online_server.py b/audio/whisper/whisper-streaming/packages/whisper_streaming/whisper_online_server.py similarity index 100% rename from whisper/whisper-streaming/packages/whisper_streaming/whisper_online_server.py rename to audio/whisper/whisper-streaming/packages/whisper_streaming/whisper_online_server.py diff --git a/whisper/whisper-streaming/requirements.txt b/audio/whisper/whisper-streaming/requirements.txt similarity index 100% rename from whisper/whisper-streaming/requirements.txt rename to audio/whisper/whisper-streaming/requirements.txt diff --git a/audio/whisper/whisper-v3-base64/README.md b/audio/whisper/whisper-v3-base64/README.md new file mode 100644 index 000000000..8027d94b2 --- /dev/null +++ b/audio/whisper/whisper-v3-base64/README.md @@ -0,0 +1,31 @@ +# Whisper V3 Base64 Input + +Transcribe audio files across multiple languages. + +| Property | Value | +|----------|-------| +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/whisper/whisper-v3-truss-base64/config.yaml b/audio/whisper/whisper-v3-base64/config.yaml similarity index 95% rename from whisper/whisper-v3-truss-base64/config.yaml rename to audio/whisper/whisper-v3-base64/config.yaml index 839fb552d..efadf3019 100644 --- a/whisper/whisper-v3-truss-base64/config.yaml +++ b/audio/whisper/whisper-v3-base64/config.yaml @@ -5,6 +5,7 @@ external_data: url: https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt external_package_dirs: [] model_metadata: + repo_id: "openai/whisper-large-v3" avatar_url: https://cdn.baseten.co/production/static/openai.png cover_image_url: https://cdn.baseten.co/production/static/whisper.png example_model_input: diff --git a/comfyui-truss/model/__init__.py b/audio/whisper/whisper-v3-base64/model/__init__.py similarity index 100% rename from comfyui-truss/model/__init__.py rename to audio/whisper/whisper-v3-base64/model/__init__.py diff --git a/whisper/whisper-v3-truss-base64/model/model.py b/audio/whisper/whisper-v3-base64/model/model.py similarity index 100% rename from whisper/whisper-v3-truss-base64/model/model.py rename to audio/whisper/whisper-v3-base64/model/model.py diff --git a/audio/whisper/whisper-v3/README.md b/audio/whisper/whisper-v3/README.md new file mode 100644 index 000000000..649772e18 --- /dev/null +++ b/audio/whisper/whisper-v3/README.md @@ -0,0 +1,31 @@ +# Whisper V3 + +Transcribe audio files across multiple languages. + +| Property | Value | +|----------|-------| +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/whisper/whisper-v3-truss/config.yaml b/audio/whisper/whisper-v3/config.yaml similarity index 95% rename from whisper/whisper-v3-truss/config.yaml rename to audio/whisper/whisper-v3/config.yaml index 5baae8918..880187efa 100644 --- a/whisper/whisper-v3-truss/config.yaml +++ b/audio/whisper/whisper-v3/config.yaml @@ -5,6 +5,7 @@ external_data: url: https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt external_package_dirs: [] model_metadata: + repo_id: "openai/whisper-large-v3" avatar_url: https://cdn.baseten.co/production/static/openai.png cover_image_url: https://cdn.baseten.co/production/static/whisper.png example_model_input: diff --git a/control-net-qrcode/model/__init__.py b/audio/whisper/whisper-v3/model/__init__.py similarity index 100% rename from control-net-qrcode/model/__init__.py rename to audio/whisper/whisper-v3/model/__init__.py diff --git a/whisper/whisper-v3-truss/model/model.py b/audio/whisper/whisper-v3/model/model.py similarity index 100% rename from whisper/whisper-v3-truss/model/model.py rename to audio/whisper/whisper-v3/model/model.py diff --git a/audio/whisper/whisperx/README.md b/audio/whisper/whisperx/README.md new file mode 100644 index 000000000..e9b826ee9 --- /dev/null +++ b/audio/whisper/whisperx/README.md @@ -0,0 +1,34 @@ +# whisperX + +Deploy whisperX for speech-to-text transcription on Baseten. + +| Property | Value | +|----------|-------| +| Task | Speech-to-text | +| Engine | Custom (Truss) | +| GPU | L4 | +| Python | py310 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "audio_file": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- Base image: `runpod/pytorch:2.1.1-py3.10-cuda12.1.1-devel-ubuntu22.04` +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/whisper/whisperx-truss/config.yaml b/audio/whisper/whisperx/config.yaml similarity index 90% rename from whisper/whisperx-truss/config.yaml rename to audio/whisper/whisperx/config.yaml index d662f9201..05651f74d 100644 --- a/whisper/whisperx-truss/config.yaml +++ b/audio/whisper/whisperx/config.yaml @@ -1,9 +1,11 @@ +description: "whisperX for speech-to-text transcription" base_image: image: runpod/pytorch:2.1.1-py3.10-cuda12.1.1-devel-ubuntu22.04 python_executable_path: /usr/bin/python environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "openai/whisper-large-v3" example_model_input: audio_file: https://cdn.baseten.co/docs/production/Gettysburg.mp3 model_name: whisperX diff --git a/deepfloyd-xl/model/__init__.py b/audio/whisper/whisperx/model/__init__.py similarity index 100% rename from deepfloyd-xl/model/__init__.py rename to audio/whisper/whisperx/model/__init__.py diff --git a/whisper/whisperx-truss/model/model.py b/audio/whisper/whisperx/model/model.py similarity index 100% rename from whisper/whisperx-truss/model/model.py rename to audio/whisper/whisperx/model/model.py diff --git a/audio/xtts-streaming/README.md b/audio/xtts-streaming/README.md new file mode 100644 index 000000000..0121118b6 --- /dev/null +++ b/audio/xtts-streaming/README.md @@ -0,0 +1,29 @@ +# XTTS Streaming - High Performance + +Deploy XTTS Streaming - High Performance for text-to-speech on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text-to-speech | +| Engine | Custom (Truss) | +| GPU | H100 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"text": "Hello, this is a test of text to speech."}' +``` + +## Configuration highlights + +- Base image: `htrivedi05/xtts-streaming` +- Environment variables: `COQUI_TOS_AGREED` diff --git a/xtts-streaming/config.yaml b/audio/xtts-streaming/config.yaml similarity index 61% rename from xtts-streaming/config.yaml rename to audio/xtts-streaming/config.yaml index 95acd151f..1a7e25a0b 100644 --- a/xtts-streaming/config.yaml +++ b/audio/xtts-streaming/config.yaml @@ -1,10 +1,13 @@ +description: "XTTS Streaming - High Performance for text-to-speech" base_image: image: htrivedi05/xtts-streaming python_executable_path: /opt/conda/bin/python environment_variables: COQUI_TOS_AGREED: '1' external_package_dirs: [] -model_metadata: {} +model_metadata: + repo_id: "coqui/XTTS-v2" + example_model_input: {"text": "Hello, how are you today?", "language": "en"} model_name: XTTS Streaming - High Performance resources: accelerator: H100 diff --git a/deepspeed-mii/model/__init__.py b/audio/xtts-streaming/model/__init__.py similarity index 100% rename from deepspeed-mii/model/__init__.py rename to audio/xtts-streaming/model/__init__.py diff --git a/xtts-streaming/model/model.py b/audio/xtts-streaming/model/model.py similarity index 100% rename from xtts-streaming/model/model.py rename to audio/xtts-streaming/model/model.py diff --git a/xtts-streaming/requirements.txt b/audio/xtts-streaming/requirements.txt similarity index 100% rename from xtts-streaming/requirements.txt rename to audio/xtts-streaming/requirements.txt diff --git a/audio/xtts-v2/README.md b/audio/xtts-v2/README.md new file mode 100644 index 000000000..5fb0fb6f5 --- /dev/null +++ b/audio/xtts-v2/README.md @@ -0,0 +1,33 @@ +# XTTS V2 + +Deploy XTTS V2 for text-to-speech on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text-to-speech | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "language": "en", + "speaker_voice": "Claribel Dervla", + "text": "Kurt watched the incoming Pelicans. The blocky jet-powered craft were so distant they were only specks against the setting sun. He hit the magnification on his faceplate and saw lines of fire tracing their reentry vectors. They would touch down in three minutes." +}' +``` + +## Configuration highlights + +- Environment variables: `COQUI_TOS_AGREED` diff --git a/xtts-v2-truss/config.yaml b/audio/xtts-v2/config.yaml similarity index 90% rename from xtts-v2-truss/config.yaml rename to audio/xtts-v2/config.yaml index 7ac58ab63..34d72877b 100644 --- a/xtts-v2-truss/config.yaml +++ b/audio/xtts-v2/config.yaml @@ -1,7 +1,9 @@ +description: "XTTS V2 for text-to-speech" environment_variables: COQUI_TOS_AGREED: "1" external_package_dirs: [] model_metadata: + repo_id: "coqui/XTTS-v2" example_model_input: language: en speaker_voice: Claribel Dervla diff --git a/dis-segmentation/model/__init__.py b/audio/xtts-v2/model/__init__.py similarity index 100% rename from dis-segmentation/model/__init__.py rename to audio/xtts-v2/model/__init__.py diff --git a/xtts-v2-truss/model/model.py b/audio/xtts-v2/model/model.py similarity index 100% rename from xtts-v2-truss/model/model.py rename to audio/xtts-v2/model/model.py diff --git a/audiogen-medium/README.md b/audiogen-medium/README.md deleted file mode 100644 index 6822e4eb6..000000000 --- a/audiogen-medium/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# AudioGen Truss - -This repository packages [AudioGen](https://github.com/facebookresearch/audiocraft/) as a [Truss](https://truss.baseten.co). - -AudioGen is a simple and controllable model for audio generation developed by Facebook AI Research. - -## Deploying AudioGen - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd audiogen-medium-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `audiogen-medium-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -We found this model runs reasonably fast on A10Gs; you can configure the hardware you'd like in the config.yaml. - -```yaml ---- -resources: - cpu: "3" - memory: 14Gi - use_gpu: true - accelerator: A10G -``` - -## Invoking AudioGen - -AudioGen takes a list of prompts and a duration in seconds. It will generate one clip per prompt and return each clip as a base64 encoded WAV file. - -```sh -truss predict -d '{"prompts": ['dog barking', 'sirene of an emergency vehicle', 'footsteps in a corridor'], "duration": 8}' -``` - -```python - -import json -import base64 -import os, sys - -model_output = json.loads(sys.stdin.read()) - -for idx, clip in enumerate(model_output["data"]): - with open(f"clip_{idx}.wav", "wb") as f: - f.write(base64.b64decode(clip)) -``` - -You can also invoke your model via a REST API - -``` -curl -X POST " https://app.baseten.co/models/YOUR_MODEL_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompts": ["happy rock" "energetic EDM", "sad jazz"], "duration": 8 - }' -``` - -## Model sizes - -AudioGen comes in 1 size: - -- `medium`: 1.5B model - -which is the model in this truss. diff --git a/autodesk-wala/README.md b/autodesk-wala/README.md deleted file mode 100644 index 34bd98be6..000000000 --- a/autodesk-wala/README.md +++ /dev/null @@ -1,75 +0,0 @@ -## Autodesk WaLa (single‑view image β†’ 3D) Truss - -- Status: supports only `ADSKAILab/WaLa-SV-1B` (single‑view) at the moment -- Output: OBJ (default) or SDF, base64‑encoded in the response -- License: non‑commercial per the model card - -### Prerequisites -- Baseten account and API key. -- Hugging Face access: - - Accept the model license on `ADSKAILab/WaLa-SV-1B`. - - Add a Baseten secret named `hf_access_token` with your token (read access). - -### Deploy -From this folder (`autodesk-wala`): - -```bash -# Option A: Truss CLI -truss push --publish -``` - -Notes: -- You can use the client code in `autodesk-wala/test.py` to call your endpoint and save the OBJ locally. -- WaLa source code is vendored in `packages/src/` (run `./vendor_wala.sh` to set up if missing). -- The HF token is read from the `hf_access_token` secret and exported for `hf_hub_download`. - -### Invoke (Python) -A simple client is provided in `test.py`. It reads `examples/single_view/table.png`, sends it to your deployed endpoint, saves `output.obj`, and displays it if `trimesh` + `plotly` are installed. - -```bash -export BASETEN_API_KEY=... # your Baseten API key -python autodesk-wala/test.py -``` - -### Invoke (cURL) -Replace `model_id` and `API_KEY` with your values. - -```bash -IMG_B64=$(base64 -i autodesk-wala/examples/single_view/table.png) - -curl -s -X POST "https://model-.api.baseten.co/production/predict" \ - -H "Authorization: Api-Key " \ - -H "Content-Type: application/json" \ - -d "{\ - \"image_b64\": \"${IMG_B64}\",\ - \"model_name\": \"ADSKAILab/WaLa-SV-1B\",\ - \"output_format\": \"obj\",\ - \"scale\": 1.8,\ - \"diffusion_rescale_timestep\": 5,\ - \"seed\": 42\ - }" | tee response.json - -# Decode OBJ -jq -r .obj_b64 response.json | base64 --decode > output.obj -``` - -### Request schema -- Required: - - `image_b64`: base64‑encoded RGB image (single view) -- Optional: - - `model_name`: HF repo id (defaults to `ADSKAILab/WaLa-SV-1B`) - - `output_format`: `obj` (default) or `sdf` - - `scale`: float, default `3.0` (authors often use `1.8`) - - `diffusion_rescale_timestep`: int, default `100` (authors often use `5`) - - `seed`: int, default `42` - - `target_num_faces`: int, optional mesh simplification target (server best‑effort; skipped if Open3D/libGL unavailable). For Colab‑like behavior, omit this. - -### Response -- Success: - - `obj_b64` or `sdf_b64`: base64 of the generated asset - - `data`: same base64 payload (for convenience) - - `format`: `obj` or `sdf` - - `output_path`: server‑side path where the file was written - - `time`: seconds -- Error: - - `{ "status": "error", "message": "..." }` diff --git a/chatterbox-tts/README.md b/chatterbox-tts/README.md deleted file mode 100644 index f35fb0cdc..000000000 --- a/chatterbox-tts/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Chatterbox TTS Example -## Overview -This is a basic example showing how to deploy [Chatterbox TTS](https://github.com/resemble-ai/chatterbox) on truss. - -## Running the Example -The truss endpoint is set up to accept text and an optional base64 wav string, and it returns that base64 string audio. - -For a more detailed example on how to run it, take a look at [`run_tts.py`](https://github.com/basetenlabs/truss-examples/blob/main/chatterbox-tts/run_tts.py). This script will take the `input_text`, apply the voice clone file `input/obama_8s.wav`, and output the audio file to `output/output_obama8s.wav`. - -## Custom Docker Image -Currently, this example is built from an slightly modified baseten docker image. That image installs `numpy==1.26.0` on a truss base image in order to work with `chatterbox-tts==0.1.1`. - -For more details, see [Creating a custom base image](https://docs.baseten.co/development/model/base-images#creating-a-custom-base-image). diff --git a/ci.yaml b/ci.yaml index fd1f7f45c..cdedf0eb8 100644 --- a/ci.yaml +++ b/ci.yaml @@ -1,15 +1,347 @@ +# Canonical list of all examples in the repository. +# test_all.py (Test 8) validates that every path here exists and loads with truss.load(). tests: - - 01-getting-started-bert - - 02-llm - - 03-llm-with-streaming - - 04-image-generation - - 05-speech-to-text - - 06-high-performance-cached-weights - - 10-using-system-packages - - whisper/whisper-v3-truss - - gfp-gan - - stable-diffusion/stable-diffusion-xl-1.0 - - whisper/faster-whisper-v3 - - playground-v2-aesthetic - - llama/tinyllama-1.1B-chat-v1.0 - #- qwen/BEI-qwen-qwen3-embedding-0.6b-A10G # add BEI model + # ── tutorials ────────────────────────────────────────────────────────────── + - tutorials/getting-started-bert + - tutorials/llm-basics + - tutorials/llm-streaming + - tutorials/image-generation + - tutorials/speech-to-text + - tutorials/cached-weights + - tutorials/system-packages + - tutorials/dynamic-batching + - tutorials/private-huggingface + + # ── llm ──────────────────────────────────────────────────────────────────── + # llm/cogito + - llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm + - llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm + - llm/cogito/cogito-v2-preview-llama-405B-vllm + - llm/cogito/cogito-v2-preview-llama-70B-vllm + # llm/cogvlm + - llm/cogvlm + # llm/deepseek + - llm/deepseek/deepseek-ocr + - llm/deepseek/deepseek-vl2 + - llm/deepseek/engine-deepseek-r1-distill-llama-70b + - llm/deepseek/engine-deepseek-r1-distill-llama-8b + - llm/deepseek/engine-deepseek-r1-distill-qwen-14b + - llm/deepseek/engine-deepseek-r1-distill-qwen-32b + - llm/deepseek/engine-deepseek-r1-distill-qwen-7b + # llm/falcon + - llm/falcon/falcon3-3B-trt-llm-engine-high-throughput + # llm/gemma + - llm/gemma/gemma-2-27b-it-vllm + - llm/gemma/gemma-2-9b-it-vllm + - llm/gemma/gemma-3-27b-it + # llm/llama + - llm/llama/engine-llama-3-1-70b-instruct + - llm/llama/engine-llama-3-1-8b-instruct + - llm/llama/engine-llama-3-3-70b-instruct + - llm/llama/engine-llama-3.1-405b-instruct + - llm/llama/llama-3-1-405b-instruct + - llm/llama/llama-3-1-70b-instruct + - llm/llama/llama-3-1-8b-instruct + - llm/llama/llama-3-1-8b-instruct-sglang + - llm/llama/llama-3-2-11b-vision-instruct + - llm/llama/llama-3-70b-instruct + - llm/llama/llama-3-8b-instruct + - llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm + - llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm + - llm/llama/tinyllama-1.1B-chat-v1.0 + # llm/llava + - llm/llava/llava-1.6-sgl + - llm/llava/llava-v1.5-7b + - llm/llava/llava-v1.6-34b + # llm/lora + - llm/lora/engine-lora + - llm/lora/sglang-lora + - llm/lora/vllm-lora + # llm/midnight + - llm/midnight + # llm/minimax + - llm/minimax/minimax-m2-1 + # llm/mistral + - llm/mistral/engine-devstral + - llm/mistral/engine-mistral-small-3 + - llm/mistral/engine-mixtral-8x22b-instruct + - llm/mistral/engine-mixtral-8x7b-instruct + - llm/mistral/mistral-7b + - llm/mistral/mistral-small-3.1 + - llm/mistral/mixtral-8x22b + - llm/mistral/mixtral-8x22b-trt-int8-weights-only + - llm/mistral/mixtral-8x7b-instruct-trt-llm + - llm/mistral/mixtral-8x7b-instruct-trt-llm-h100 + - llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant + - llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 + - llm/mistral/mixtral-8x7b-instruct-vllm + - llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2 + - llm/mistral/pixtral-12b + # llm/nemotron + - llm/nemotron/llama-3-1-nemotron-70b-instruct + - llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1 + - llm/nemotron/llama-nemoretriever-colembed-3b-v1 + - llm/nemotron/nemotron-3-nano + - llm/nemotron/nemotron-3-nano-nvfp4 + - llm/nemotron/nemotron-nano-12b-v2-vl-bf16 + - llm/nemotron/nemotron-ultra-253b + # llm/nsql + - llm/nsql + # llm/openai + - llm/openai/gpt-oss-120b + - llm/openai/gpt-oss-20b + # llm/personaplex + - llm/personaplex-7b-v1 + # llm/phi + - llm/phi/phi-3-mini-128k-instruct + - llm/phi/phi-3-mini-4k-instruct + - llm/phi/phi-3.5-mini + # llm/qwen + - llm/qwen/engine-qwen-2-5-14b-coder-instruct + - llm/qwen/engine-qwen-2-5-14b-instruct + - llm/qwen/engine-qwen-2-5-32b-coder-instruct + - llm/qwen/engine-qwen-2-5-32b-instruct + - llm/qwen/engine-qwen-2-5-3b-instruct + - llm/qwen/engine-qwen-2-5-72b-instruct + - llm/qwen/engine-qwen-2-5-72b-math-instruct + - llm/qwen/engine-qwen-2-5-7b-coder-instruct + - llm/qwen/engine-qwen-2-5-7b-instruct + - llm/qwen/engine-qwen-2-5-7b-math-instruct + - llm/qwen/engine-qwen-3-06b + - llm/qwen/engine-qwen-3-30b-a3b-instruct-2507 + - llm/qwen/engine-qwen-3-32b + - llm/qwen/engine-qwen-3-4b + - llm/qwen/qwen-3-235B-A22B-instruct-2507-trt + - llm/qwen/qwen-3-235B-sglang + - llm/qwen/qwen-3-30B-A3-coder + - llm/qwen/qwen-3-30B-A3-sglang + - llm/qwen/qwen-3-30B-A3-vllm + - llm/qwen/qwen-3-32B-sglang + - llm/qwen/qwen-3-next-80B-A3-instruct-sglang + - llm/qwen/qwen-3-next-80B-A3-thinking-sglang + - llm/qwen/qwen-3-vl-30b-a3b-instruct + - llm/qwen/qwen-3-vl-30b-a3b-thinking + - llm/qwen/qwen-3-vl-32b + - llm/qwen/qwen-coder-next + - llm/qwen/qwen-image + - llm/qwen/qwen-vl + # llm/seed + - llm/seed/seed-llm + # llm/z-ai + - llm/z-ai/glm-4-5-air-fp8 + - llm/z-ai/glm-4-5-fp8 + - llm/z-ai/glm-4-5-v + - llm/z-ai/glm-4-6-fp8 + - llm/z-ai/glm-4-7-flash + + # ── embeddings ───────────────────────────────────────────────────────────── + # embeddings/bei + - embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding + - embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding + - embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding + - embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base + - embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8 + - embeddings/bei/baai-bge-en-icl-embedding-fp8 + - embeddings/bei/baai-bge-large-en-v1.5-embedding + - embeddings/bei/baai-bge-m3-embedding-dense + - embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding + - embeddings/bei/baai-bge-reranker-large + - embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large + - embeddings/bei/baai-bge-reranker-v2-m3-multilingual + - embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8 + - embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8 + - embeddings/bei/google-embeddinggemma-300m + - embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8 + - embeddings/bei/intfloat-multilingual-e5-large-instruct + - embeddings/bei/jina-ai-jina-embeddings-v2-base-en + - embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8 + - embeddings/bei/jinaai-jina-embeddings-v2-base-code + - embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding + - embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding + - embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8 + - embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8 + - embeddings/bei/ncbi-medcpt-cross-encoder-reranker + - embeddings/bei/ner-bert-base-ner-uncased + - embeddings/bei/nomic-ai-nomic-embed-code-fp8 + - embeddings/bei/nomic-ai-nomic-embed-text-v1.5 + - embeddings/bei/nomic-ai-nomic-embed-text-v2-moe + - embeddings/bei/nvidia-llama-embed-nemotron-8b + - embeddings/bei/nvidia-llama-nemotron-embed-1b-v2 + - embeddings/bei/papluca-xlm-roberta-base-language-detection-classification + - embeddings/bei/qwen-qwen3-embedding-0.6b-fp8 + - embeddings/bei/qwen-qwen3-embedding-4b-fp4 + - embeddings/bei/qwen-qwen3-embedding-4b-fp8 + - embeddings/bei/qwen-qwen3-embedding-8b-fp8 + - embeddings/bei/qwen-qwen3-reranker-0.6b-fp8 + - embeddings/bei/qwen-qwen3-reranker-4b-fp8 + - embeddings/bei/qwen-qwen3-reranker-8b-fp4 + - embeddings/bei/qwen-qwen3-reranker-8b-fp8 + - embeddings/bei/redis-langcache-embed-v2 + - embeddings/bei/salesforce-sfr-embedding-mistral-fp8 + - embeddings/bei/samlowe-roberta-base-go_emotions-classification + - embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding + - embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8 + - embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0 + - embeddings/bei/tanaos-tanaos-ner-v1 + - embeddings/bei/taylorai-bge-micro-v2 + - embeddings/bei/voyageai-voyage-4-nano + - embeddings/bei/whereisai-uae-large-v1-embedding + # embeddings/tei + - embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding + - embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding + - embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding + - embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base + - embeddings/tei/baai-bge-reranker-large + - embeddings/tei/google-embeddinggemma-300m + - embeddings/tei/intfloat-multilingual-e5-large-instruct + - embeddings/tei/jina-ai-jina-embeddings-v2-base-en + - embeddings/tei/jinaai-jina-embeddings-v2-base-code + - embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding + - embeddings/tei/nomic-ai-nomic-embed-text-v1.5 + - embeddings/tei/nomic-ai-nomic-embed-text-v2-moe + - embeddings/tei/redis-langcache-embed-v2 + - embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding + - embeddings/tei/taylorai-bge-micro-v2 + # embeddings/other + - embeddings/clip + - embeddings/text-embeddings-inference + + # ── image ────────────────────────────────────────────────────────────────── + - image/comfyui + - image/comfyui/examples/anime-style-transfer + - image/control-net-qrcode + - image/deepfloyd-xl + - image/dis-segmentation + - image/flux-dev-trt-b200 + - image/flux/dev + - image/flux/schnell + - image/fotographer/zenctrl + - image/gfp-gan + - image/image-segmentation + - image/ip-adapter + - image/magic-animate + - image/playground-v2-aesthetic + - image/sana/sana-1600m + - image/sana/sana-600m + - image/segment-anything + - image/stable-diffusion/dreamshaper-lcm + - image/stable-diffusion/playground-v2-trt + - image/stable-diffusion/sd-textual-inversion + - image/stable-diffusion/sd-turbo + - image/stable-diffusion/sdxl-controlnet + - image/stable-diffusion/sdxl-controlnet-canny + - image/stable-diffusion/sdxl-controlnet-depth + - image/stable-diffusion/sdxl-lightning + - image/stable-diffusion/sdxl-lora + - image/stable-diffusion/sdxl-lora-swapping + - image/stable-diffusion/sdxl-turbo + - image/stable-diffusion/stable-diffusion + - image/stable-diffusion/stable-diffusion-3-medium + - image/stable-diffusion/stable-diffusion-inpainting-trt + - image/stable-diffusion/stable-diffusion-xl-1.0 + - image/stable-diffusion/stable-diffusion-xl-1.0-trt + - image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100 + - image/stable-diffusion/stable-video-diffusion + + # ── audio ────────────────────────────────────────────────────────────────── + - audio/audiogen-medium + - audio/chatterbox-tts + - audio/kokoro + - audio/metavoice-1b + - audio/musicgen-large + - audio/musicgen-melody + - audio/nvidia-parakeet + - audio/orpheus-3b-websockets + - audio/orpheus-best-performance + - audio/piper-tts + - audio/qwen-asr + - audio/qwen-omni + - audio/qwen-omni-thinker + - audio/sesame-csm-1b + - audio/ultravox + - audio/voxtral-streaming-4b + - audio/whisper/faster-whisper-small + - audio/whisper/faster-whisper-v2 + - audio/whisper/faster-whisper-v3 + - audio/whisper/whisper-openai + - audio/whisper/whisper-streaming + - audio/whisper/whisper-v3 + - audio/whisper/whisper-v3-base64 + - audio/whisper/whisperx + - audio/xtts-streaming + - audio/xtts-v2 + + # ── optimized ────────────────────────────────────────────────────────────── + # optimized/bisv2 + - optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4 + - optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only + - optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8 + - optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4 + - optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4 + - optimized/bisv2/nvidia-qwen3-30b-a3b-fp4 + - optimized/bisv2/nvidia-qwen3-8b-fp4 + - optimized/bisv2/qwen-qwen2.5-coder-7b-instruct + - optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4 + - optimized/bisv2/qwen-qwen3-32b-fp4 + - optimized/bisv2/qwen-qwen3-4b-fp8 + # optimized/briton + - optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8 + - optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8 + - optimized/briton/google-gemma-3-1b-it + - optimized/briton/google-gemma-3-270m-it + - optimized/briton/google-gemma-3-27b-it + - optimized/briton/google-gemma-3-27b-it-speculative-lookahead + - optimized/briton/meta-llama-llama-3.1-405b-fp8 + - optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8 + - optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8 + - optimized/briton/meta-llama-llama-3.2-3b-instruct + - optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8 + - optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8 + - optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4 + - optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8 + - optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8 + - optimized/briton/microsoft-phi-4-fp8 + - optimized/briton/mistralai-mistral-7b-instruct-v0.3 + - optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8 + - optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8 + - optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8 + - optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only + - optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8 + - optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8 + - optimized/briton/qwen-qwen3-30b-a3b + - optimized/briton/qwen-qwen3-30b-a3b-fp8 + - optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8 + - optimized/briton/qwen-qwen3-32b-fp4 + - optimized/briton/qwen-qwen3-32b-fp4-mlp-only + - optimized/briton/qwen-qwen3-32b-fp8 + - optimized/briton/qwen-qwen3-8b-min-latency-fp8 + - optimized/briton/qwen-qwq-32b-reasoning-fp8 + - optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8 + - optimized/briton/tiiuae-falcon3-10b-instruct-fp8 + + # ── infrastructure ───────────────────────────────────────────────────────── + - infrastructure/autodesk-wala + - infrastructure/binocular + - infrastructure/custom-engine-builder-control + - infrastructure/custom-server/deepseek-v2-5-instruct-sglang + - infrastructure/custom-server/infinity-embedding-server + - infrastructure/custom-server/llama3-70b-instruct-lmdeploy + - infrastructure/custom-server/llama3-70b-instruct-sglang + - infrastructure/custom-server/llama3-8b-instruct-lmdeploy + - infrastructure/custom-server/llama3-8b-instruct-sglang + - infrastructure/custom-server/pixtral-12b + - infrastructure/custom-server/ultravox-0.4 + - infrastructure/custom-server/ultravox-0.5-8b + - infrastructure/custom-server/ultravox-0.6-70b + - infrastructure/custom-server/voxtral-mini-3b-2507 + - infrastructure/custom-server/voxtral-small-24b-2507 + - infrastructure/grpc + - infrastructure/jsonformatter + - infrastructure/layoutlm-document-qa + - infrastructure/llama-cpp-server + - infrastructure/metrics/datadog + - infrastructure/model-cache + - infrastructure/multiprocessing + - infrastructure/ngram-speculator/truss + - infrastructure/ngram-speculator/trussless + - infrastructure/paddlepaddle/paddleocr-vl diff --git a/cogito/README.md b/cogito/README.md deleted file mode 100644 index f3336b2bb..000000000 --- a/cogito/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# Cogito v2 Preview Models - -This directory contains [Truss](https://truss.baseten.co/) configurations for deploying Cogito v2 preview models using vLLM's OpenAI-compatible server. These models feature powerful tool calling and reasoning capabilities, as detailed in the [Cogito v2 research blog post](https://www.deepcogito.com/research/cogito-v2-preview). - -## Available Models - -| Model | Size | Architecture | GPU Requirements | -|-------|------|--------------|------------------| -| [Cogito v2 Llama 70B](./cogito-v2-preview-llama-70B-vllm/) | 70B | Dense | 2x H100 | -| [Cogito v2 Llama 109B MoE](./cogito-v2-preview-llama-109B-MoE-vllm/) | 109B | MoE (Mixture of Experts) | 4x H100 | -| [Cogito v2 Llama 405B](./cogito-v2-preview-llama-405B-vllm/) | 405B | Dense | 8x B200 | -| [Cogito v2 DeepSeek 671B MoE](./cogito-v2-preview-deepseek-671B-MoE-vllm/) | 671B | MoE (Mixture of Experts) | 8x B200 | - - - - -## Prerequisites - -Before deploying any of these models, ensure you have: - -1. **Baseten Account**: Sign up at [app.baseten.co/signup](https://app.baseten.co/signup) -2. **API Key**: Get your API key from [app.baseten.co/settings/api_keys](https://app.baseten.co/settings/api_keys) -3. **Truss Installation**: Install the latest version: `pip install --upgrade truss` -4. **Hugging Face Token**: Retrieve from [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens) -5. **Baseten Secret**: Set your HF token as a Baseten secret with key `hf_access_token` at [app.baseten.co/settings/secrets](https://app.baseten.co/settings/secrets) - -## Deployment - -### Quick Start - -1. Clone this repository: -```bash -git clone https://github.com/basetenlabs/truss-examples.git -cd cogito -``` - -2. Navigate to your desired model directory: -```bash -cd cogito-v2-preview-llama-70B-vllm # or any other model -``` - -3. Deploy the model: -```bash -truss push --publish -``` - -### GPU Requirements - -- **B200 GPUs**: Required for 405B and 671B models (contact [support@baseten.co](mailto:support@baseten.co) before deploying) -- **H100 GPUs**: Required for 70B and 109B MoE models - -## API Usage - -All models follow the OpenAI ChatCompletion format. Here's an example using the Python client for tool calling: - -```python -from openai import OpenAI - -# Replace with your model ID after deployment -model_id = "your-model-id" # e.g. "yqvy46gq" -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -# Example tool calling -def get_temperature_in_celsius(location=None): - return 22 - -tools = [ - { - "type": "function", - "function": { - "name": "get_temperature_in_celsius", - "description": "Get the current temperature in celsius.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get the temperature for." - } - }, - "required": ["location"] - } - } - } -] - -# Chat completion with tool calling -response = client.chat.completions.create( - model="llama", # or "deepseek" for DeepSeek model - messages=[ - { - "role": "user", - "content": "What is today's temperature in celsius? I'm in Paris." - } - ], - tools=tools, - max_tokens=1000, - temperature=0.6 -) - -print(response.json()) -``` - - -## Support - -- **Documentation**: [Truss documentation](https://truss.baseten.co) -- **Issues**: Open an issue in this repository -- **Support**: Contact [support@baseten.co](mailto:support@baseten.co) -- **Research**: [Cogito v2 research blog](https://www.deepcogito.com/research/cogito-v2-preview) - -## License - -These models are subject to the respective licenses of the underlying model weights. Please refer to the Hugging Face model pages for specific licensing information. diff --git a/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md b/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md deleted file mode 100644 index 66259a566..000000000 --- a/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# Cogito v2 DeepSeek 671B MoE Truss (vLLM) - -Cogito's DeepSeek-based 671B MoE model has powerful tool calling and reasoning capabilities. See this [blog post](https://www.deepcogito.com/research/cogito-v2-preview). - -This is a [Truss](https://truss.baseten.co/) to deploy the model using the vLLM OpenAI Compatible server. This model requires 8x B200 GPUs to deploy. Users should contact [support@baseten.co](mailto:support@baseten.co) before deploying. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd cogito/cogito-v2-preview-deepseek-671B-MoE-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Note that you will *not* be able to successfully deploy the model without doing this. - -With `cogito-v2-preview-deepseek-671B-MoE-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -model_id = "your-model-id" # Replace with your model ID - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -def get_temperature_in_celsius(location=None): - return 22 - -tools = [ - { - "type": "function", - "function": { - "name": "get_temperature_in_celsius", - "description": "Get the current temperature in celsius.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get the temperature for." - } - }, - "required": [ - "location" - ] - } - } - } -] - -# Example usage of the OpenAI client to use a tool call -response = client.chat.completions.create( - model="deepseek", - messages=[ - { - "role": "user", - "content": "What is today's temperature in celsius? I'm in Paris." - } - ], - tools=tools, -) - -print(response.json()) -``` - -## Model Details - -- **Model**: Cogito v2 Preview DeepSeek 671B MoE FP8 -- **Architecture**: Mixture of Experts (MoE) -- **GPU Requirements**: 8x B200 -- **Tool Call Parser**: deepseek_v3 -- **Tensor Parallel Size**: 8 -- **GPU Memory Utilization**: 90% - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our [support team](mailto:support@baseten.co). diff --git a/cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md b/cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md deleted file mode 100644 index 073c5b613..000000000 --- a/cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# Cogito v2 Llama 109B MoE Truss (vLLM) - -Cogito's Llama-based 109B MoE model has powerful tool calling and reasoning capabilities. See this [blog post](https://www.deepcogito.com/research/cogito-v2-preview). - -This is a [Truss](https://truss.baseten.co/) to deploy the model using the vLLM OpenAI Compatible server. This model requires 4x H100 GPUs to deploy. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd cogito/cogito-v2-preview-llama-109B-MoE-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Note that you will *not* be able to successfully deploy the model without doing this. - -With `cogito-v2-preview-llama-109B-MoE-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -model_id = "your-model-id" # Replace with your model ID - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -def get_temperature_in_celsius(location=None): - return 22 - -tools = [ - { - "type": "function", - "function": { - "name": "get_temperature_in_celsius", - "description": "Get the current temperature in celsius.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get the temperature for." - } - }, - "required": [ - "location" - ] - } - } - } -] - -# Example usage of the OpenAI client to use a tool call -response = client.chat.completions.create( - model="llama", - messages=[ - { - "role": "user", - "content": "What is today's temperature in celsius? I'm in Paris." - } - ], - tools=tools, -) - -print(response.json()) -``` - -## Model Details - -- **Model**: Cogito v2 Preview Llama 109B MoE -- **Architecture**: Mixture of Experts (MoE) -- **GPU Requirements**: 4x H100 -- **Tool Call Parser**: llama3_json -- **Tensor Parallel Size**: 4 - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our [support team](mailto:support@baseten.co). diff --git a/cogito/cogito-v2-preview-llama-405B-vllm/README.md b/cogito/cogito-v2-preview-llama-405B-vllm/README.md deleted file mode 100644 index 2c367025a..000000000 --- a/cogito/cogito-v2-preview-llama-405B-vllm/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# Cogito v2 Llama 405B Truss (vLLM) - -Cogito's Llama-based 405B model has powerful tool calling and reasoning capabilities. See this [blog post](https://www.deepcogito.com/research/cogito-v2-preview). - -This is a [Truss](https://truss.baseten.co/) to deploy the model using the vLLM OpenAI Compatible server. This model requires 8x B200 GPUs to deploy. Users should contact [support@baseten.co](mailto:support@baseten.co) before deploying. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd cogito/cogito-v2-preview-llama-405B-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Note that you will *not* be able to successfully deploy the model without doing this. - -With `cogito-v2-preview-llama-405B-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -model_id = "your-model-id" # Replace with your model ID - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -def get_temperature_in_celsius(location=None): - return 22 - -tools = [ - { - "type": "function", - "function": { - "name": "get_temperature_in_celsius", - "description": "Get the current temperature in celsius.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get the temperature for." - } - }, - "required": [ - "location" - ] - } - } - } -] - -# Example usage of the OpenAI client to use a tool call -response = client.chat.completions.create( - model="llama", - messages=[ - { - "role": "user", - "content": "What is today's temperature in celsius? I'm in Paris." - } - ], - tools=tools, -) - -print(response.json()) -``` - -## Model Details - -- **Model**: Cogito v2 Preview Llama 405B -- **Architecture**: Dense transformer -- **GPU Requirements**: 8x B200 -- **Tool Call Parser**: llama3_json -- **Features**: Prefix caching, chunked prefill -- **Tensor Parallel Size**: 8 - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our [support team](mailto:support@baseten.co). diff --git a/cogito/cogito-v2-preview-llama-70B-vllm/README.md b/cogito/cogito-v2-preview-llama-70B-vllm/README.md deleted file mode 100644 index 6e8771728..000000000 --- a/cogito/cogito-v2-preview-llama-70B-vllm/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# Cogito v2 Llama 70B Truss (vLLM) - -Cogito's Llama-based 70B model has powerful tool calling and reasoning capabilities. See this [blog post](https://www.deepcogito.com/research/cogito-v2-preview). - -This is a [Truss](https://truss.baseten.co/) to deploy the model using the vLLM OpenAI Compatible server. This model requires 2x H100 GPUs to deploy. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd cogito/cogito-v2-preview-llama-70B-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Note that you will *not* be able to successfully deploy the model without doing this. - -With `cogito-v2-preview-llama-70B-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -model_id = "your-model-id" # Replace with your model ID - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -def get_temperature_in_celsius(location=None): - return 22 - -tools = [ - { - "type": "function", - "function": { - "name": "get_temperature_in_celsius", - "description": "Get the current temperature in celsius.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get the temperature for." - } - }, - "required": [ - "location" - ] - } - } - } -] - -# Example usage of the OpenAI client to use a tool call -response = client.chat.completions.create( - model="llama", - messages=[ - { - "role": "user", - "content": "What is today's temperature in celsius? I'm in Paris." - } - ], - tools=tools, -) - -print(response.json()) -``` - -## Model Details - -- **Model**: Cogito v2 Preview Llama 70B -- **Architecture**: Dense transformer -- **GPU Requirements**: 2x H100 -- **Tool Call Parser**: llama3_json -- **GPU Memory Utilization**: 95% - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our [support team](mailto:support@baseten.co). diff --git a/cogvlm/README.md b/cogvlm/README.md deleted file mode 100644 index ff38efe7e..000000000 --- a/cogvlm/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# CogVLM Truss - -This repository packages [CogVLM](https://github.com/THUDM/CogVLM) as a [Truss](https://truss.baseten.co/). - -CogVLM is a highly performant open-source vision language model with capabilities similar to GPT-4V. - -## Deploying CogVLM - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd cogvlm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `cogvlm` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking CogVLM - -CogVLM takes in two inputs, a `query` and a base64 encoded `image`. CogVLM will respond to the `query` conditioned on the `image`. The output is a JSON blob with a single key, `result`, that answers the `query`. - - -```sh -truss predict -d '{"query": "Describe this picture in detail.", "image": "data:image/png;base64,iVBORw0KGgoA..."}' -``` - -You can also invoke your model via a REST API -``` -curl -X POST https://app.baseten.co/model_versions//predict \ - -H "Content-Type: application/json" \ - -d '{ - "query": "Describe this picture in detail.", - "image": "data:image/png;base64,iVBORw0KGgoA..." - }' -``` diff --git a/comfyui-truss/README.md b/comfyui-truss/README.md deleted file mode 100644 index 66e058b29..000000000 --- a/comfyui-truss/README.md +++ /dev/null @@ -1,398 +0,0 @@ -## ComfyUI Truss - -This truss is designed to allow ComfyUI users to easily convert their workflows into a production grade API service. - -## Exporting the ComfyUI workflow - -This Truss is designed to run a Comfy UI workflow that is in the form of a JSON file. - -Inside ComfyUI, you can save workflows as a JSON file. However, the regular JSON format that ComfyUI uses will not work. Instead, the workflow has to be saved in the API format. Here is how you can do that: - -First, go to ComfyUI and click on the gear icon for the project - -![gear_icon](../assets/comfyui-screenshot-1.png) - -Next, checkmark the box which says `Enable Dev Mode Options` - -![enable_dev_mode_options](../assets/comfyui-screenshot-2.png) - -Now, if you go back to the project you will see a new option called `Save (API Format)`. This is the one you want to use to save your workflow. Using this method you can save any ComfyUI workflow as a JSON file in the API format. - -![save_api_format](../assets/comfyui-screenshot-3.png) - - -## Setting up the project - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd comfyui-truss -``` - -For your ComfyUI workflow, you probably used one or more models. Those models need to be defined inside truss. From the root of the truss project, open the file called `config.yaml`. In this file we will modify an element called `build_commands`. Build commands will allow you to run docker commands at build time. You can use this feature to download model weights and install custom nodes. - -```yaml -build_commands: -- git clone https://github.com/comfyanonymous/ComfyUI.git -- cd ComfyUI && git checkout b1fd26fe9e55163f780bf9e5f56bf9bf5f035c93 && pip install -r requirements.txt -- cd ComfyUI/custom_nodes && git clone https://github.com/LykosAI/ComfyUI-Inference-Core-Nodes --recursive && cd ComfyUI-Inference-Core-Nodes && pip install -e .[cuda12] -- cd ComfyUI/custom_nodes && git clone https://github.com/ZHO-ZHO-ZHO/ComfyUI-Gemini --recursive && cd ComfyUI-Gemini && pip install -r requirements.txt -- cd ComfyUI/custom_nodes && git clone https://github.com/kijai/ComfyUI-Marigold --recursive && cd ComfyUI-Marigold && pip install -r requirements.txt -- cd ComfyUI/custom_nodes && git clone https://github.com/omar92/ComfyUI-QualityOfLifeSuit_Omar92 --recursive -- cd ComfyUI/custom_nodes && git clone https://github.com/Fannovel16/comfyui_controlnet_aux --recursive && cd comfyui_controlnet_aux && pip install -r requirements.txt -- cd ComfyUI/models/controlnet && wget -O control-lora-canny-rank256.safetensors https://huggingface.co/stabilityai/control-lora/resolve/main/control-LoRAs-rank256/control-lora-canny-rank256.safetensors -- cd ComfyUI/models/controlnet && wget -O control-lora-depth-rank256.safetensors https://huggingface.co/stabilityai/control-lora/resolve/main/control-LoRAs-rank256/control-lora-depth-rank256.safetensors -- cd ComfyUI/models/checkpoints && wget -O dreamshaperXL_v21TurboDPMSDE.safetensors https://civitai.com/api/download/models/351306 -- cd ComfyUI/models/loras && wget -O StudioGhibli.Redmond-StdGBRRedmAF-StudioGhibli.safetensors https://huggingface.co/artificialguybr/StudioGhibli.Redmond-V2/resolve/main/StudioGhibli.Redmond-StdGBRRedmAF-StudioGhibli.safetensors -environment_variables: {} -external_package_dirs: [] -model_metadata: {} -model_name: comfy build commands -python_version: py310 -requirements: - - websocket-client==1.6.4 - - accelerate==0.23.0 - - opencv-python -resources: - accelerator: A100 - use_gpu: true -secrets: {} -system_packages: - - wget - - ffmpeg - - libgl1-mesa-glx -``` - -Here is a breakdown of the actions happenning in `build_commands`. First, we are cloning the ComfyUI repository, checking out a specific commit, and installing the required python packages to run ComfyUI. Next, we use the `cd` commmand ensure all custom nodes are downloaded within the `custom_nodes` directory, and their respective requirements are installed. Similarly for the checkpoints, we use the `wget` utility to download the checkpoints and place them in the appropriate directories within ComfyUI. - -Every line under `build_commands` effectively does a `RUN` inside the dockerfile. The benefit of using this feature is that your model weights and custom nodes get cached during the docker build stage. So when your model deploys, it can access the cached weights directly which reduces the cold-start time. - -We also need to place the JSON workflow from step 1 inside the data directory. In the data directory create an open a file called `data/comfy_ui_workflow.json`. Copy and paste the entire JSON workflow that we saved in step 1 into this file. - -In the JSON workflow file, there might be some inputs such as the positive prompt or negative prompt that are hard coded. We want these inputs to be dynamically sent to the model, so we can use handlebars to templatize them. Here is an example of a JSON workflow with templatized inputs: - -```json -{ - "6": { - "inputs": { - "text": "{{positive_prompt}}", - "clip": [ - "14", - 1 - ] - }, - "class_type": "CLIPTextEncode" - }, - "7": { - "inputs": { - "text": "{{negative_prompt}}", - "clip": [ - "14", - 1 - ] - }, - "class_type": "CLIPTextEncode" - }, - "11": { - "inputs": { - "image": "{{controlnet_image}}", - "choose file to upload": "image" - }, - "class_type": "LoadImage" - }, - "12": { - "inputs": { - "control_net_name": "diffusers_xl_canny_full.safetensors" - }, - "class_type": "ControlNetLoader" - }, - "14": { - "inputs": { - "ckpt_name": "sd_xl_base_1.0.safetensors" - }, - "class_type": "CheckpointLoaderSimple" - }, - "15": { - "inputs": { - "images": [ - "16", - 0 - ] - }, - "class_type": "PreviewImage" - }, - "18": { - "inputs": { - "images": [ - "8", - 0 - ] - }, - "class_type": "PreviewImage" - } -} -``` - -This is not the entire JSON workflow file, but the nodes 6, 7, and 11 accept variable inputs. You can do this by using the handlebars format of `{{variable_name_here}}`. - -## Custom Nodes -If your workflow uses custom nodes you add it to the `build_commands` in the `config.yaml` file. Let's take an example. Suppose you want to add the [UltimateSDUpscale](https://github.com/ssitu/ComfyUI_UltimateSDUpscale) custom node. Inside your `config.yaml` you can define it like so: - -```yaml -build_commands: -- cd ComfyUI/custom_nodes && git clone https://github.com/ssitu/ComfyUI_UltimateSDUpscale --recursive -``` - -Each command in `build_commands` is independent of the previous command. So you need to execute this part `cd ComfyUI/custom_nodes` each time you want to install a new custom node. - -Once you have both the `data/comfy_ui_workflow.json` and `config.yaml` set up correctly we can begin deployment. - -## Deployment - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `comfyui-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Model Inference - -When an inference request is sent to the Truss, the `comfy_ui_workflow.json` in the data directory is sent to ComfyUI. If you recall, there are some templatized variables inside that json file using the handlebars format of `{{variable_name_here}}`. During inference time, we can dynamically pass in those templatized variables to our Truss prediction request like so: - -```python -values = { - "positive_prompt": "An igloo on a snowy day, 4k, hd", - "negative_prompt": "blurry, text, low quality", - "controlnet_image": "https://storage.googleapis.com/logos-bucket-01/baseten_logo.png" -} -``` - -Just be sure that the variable names in the `comfy_ui_workflow.json` template match the names inside the values object. - -Here is a complete example of how you make a prediction request to your truss in python: - -This is the content of `data/comfy_ui_workflow.json`: -```json -sdxl_controlnet_workflow = { - "3": { - "inputs": { - "seed": 972197629127129, - "steps": 40, - "cfg": 7, - "sampler_name": "euler", - "scheduler": "normal", - "denoise": 1, - "model": [ - "14", - 0 - ], - "positive": [ - "10", - 0 - ], - "negative": [ - "7", - 0 - ], - "latent_image": [ - "5", - 0 - ] - }, - "class_type": "KSampler" - }, - "5": { - "inputs": { - "width": 1024, - "height": 1024, - "batch_size": 1 - }, - "class_type": "EmptyLatentImage" - }, - "6": { - "inputs": { - "text": "{{positive_prompt}}", - "clip": [ - "14", - 1 - ] - }, - "class_type": "CLIPTextEncode" - }, - "7": { - "inputs": { - "text": "{{negative_prompt}}", - "clip": [ - "14", - 1 - ] - }, - "class_type": "CLIPTextEncode" - }, - "8": { - "inputs": { - "samples": [ - "3", - 0 - ], - "vae": [ - "14", - 2 - ] - }, - "class_type": "VAEDecode" - }, - "10": { - "inputs": { - "strength": 0.6, - "conditioning": [ - "6", - 0 - ], - "control_net": [ - "12", - 0 - ], - "image": [ - "16", - 0 - ] - }, - "class_type": "ControlNetApply" - }, - "11": { - "inputs": { - "image": "{{controlnet_image}}", - "choose file to upload": "image" - }, - "class_type": "LoadImage" - }, - "12": { - "inputs": { - "control_net_name": "diffusers_xl_canny_full.safetensors" - }, - "class_type": "ControlNetLoader" - }, - "14": { - "inputs": { - "ckpt_name": "sd_xl_base_1.0.safetensors" - }, - "class_type": "CheckpointLoaderSimple" - }, - "15": { - "inputs": { - "images": [ - "16", - 0 - ] - }, - "class_type": "PreviewImage" - }, - "16": { - "inputs": { - "low_threshold": 0.2, - "high_threshold": 0.6, - "image": [ - "11", - 0 - ] - }, - "class_type": "Canny" - }, - "18": { - "inputs": { - "images": [ - "8", - 0 - ] - }, - "class_type": "PreviewImage" - } -} -``` - -Here is the actual API request sent to Truss: -```python -import os -import random -import base64 -import requests - -# Set essential values -model_id = "" -baseten_api_key = "" -# Set prompts and controlnet image -values = { - "positive_prompt": "A top down view of a river through the woods", - "negative_prompt": "blurry, text, low quality", - "controlnet_image": "https://storage.googleapis.com/logos-bucket-01/baseten_logo.png", - "seed": random.randint(1, 1000000) -} -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/development/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json={"workflow_values": values} -) -# Get output image -res = res.json() -preamble = "data:image/png;base64," -output = base64.b64decode(res["result"][1]["image"].replace(preamble, "")) -# Save image to file -img_file = open("comfyui.png", 'wb') -img_file.write(output) -img_file.close() -os.system("open comfyui.png") -``` - -Here is the output of the request above: - -```json -[ - { - "node_id": "18", - "data": "base64-image-string", - "format": "png" - }, - { - "node_id": "15", - "data": "base64-image-string", - "format": "png" - } -] -``` - -The output of the model is a list of JSON objects containing the ID of the output node along with the generated image as a base64 string. - -You can also send input images as base64 strings. In the above example simply change the `values` python dictionary to look like this: - -```python -from PIL import Image -from io import BytesIO -import base64 - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -new_values = { - "positive_prompt": "A top down view of a river through the woods", - "negative_prompt": "blurry, text, low quality", - "controlnet_image": {"type": "image", "data": pil_to_b64(Image.open("my-image.jpeg"))}, - "seed": random.randint(1, 1000000) -} -``` - -When using base64 as input you need to specify the `type` so that the model can convert it to the correct data. diff --git a/control-net-qrcode/README.md b/control-net-qrcode/README.md deleted file mode 100644 index f688f487f..000000000 --- a/control-net-qrcode/README.md +++ /dev/null @@ -1,95 +0,0 @@ -# Control Net QR Code Truss - -This truss allows you to generate QR codes using Stable Diffusion and Control Net. By typing in a prompt using the same guidelines as stable diffusion, a new image gets created that combines the image from the prompt with the image of a qr code. For this truss [Stable Diffusion 1.5](https://huggingface.co/runwayml/stable-diffusion-v1-5) is used along with [Control QR Code Monster](https://huggingface.co/monster-labs/control_v1p_sd15_qrcode_monster) as the control net. - -![controlnet_qr_code](controlnet_qr_code_results.gif) - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd control-net-qrcode -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `control-net-qrcode` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -This two models combined only take up about 5 GB of VRAM so a T4 is enough for this truss. - -### API route: `predict` - -The predict route is the primary method for generating images based on a given prompt. It takes several parameters: - -- __prompt__ (required): The input text required for image generation. -- __qr_code_content__ (optional): The URL the QR code points to. -- __mask__ (optional): If a URL is not passed into qr_code_content, a custom image can be used as a mask -- __negative_prompt__ (optional, default=""): Use this to refine the image generation by discarding unwanted items. -- __guidance_scale__ (optional, default=7.5): Used to control image generation. -- __condition_scale__ (optional, default=1.2): The lower the condition_scale, the more creative the results. A higher condition_scale will result in less creative and more scannable QR Codes. -- __sampler__(optional, default="Euler a"): This controls which sampler to use resulting in more image variations. - -## Example usage - -```sh -truss predict -d '{"prompt": "A cubism painting of the Garden of Eaden with animals walking around, Andreas Rocha, matte painting concept art, a detailed matte painting", "qr_code_content": "https://www.baseten.co"}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "A cubism painting of the Garden of Eaden with animals walking around, Andreas Rocha, matte painting concept art, a detailed matte painting", "qr_code_content": "https://www.baseten.co" - }' -``` - -If you want to use a mask instead of a QR code you can call the API like so: -``` python -import requests -import base64 -from io import BytesIO -from PIL import Image -BASE64_PREAMBLE = "data:image/png;base64," - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - -input_image = Image.open("image/path/here") -b64_img = pil_to_b64(input_image) -headers = {"Authorization": f"Api-Key YOUR-API-KEY-HERE"} -data = { - "prompt": "a cubism painting of the Garden of Eaden with animals walking around, Andreas Rocha, matte painting concept art, a detailed matte painting", - "mask": b64_img -} -res = requests.post("https://app.baseten.co/model_versions/MODEL_VERSION/predict", headers=headers, json=data) -pil_img = b64_to_pil(output.get("model_output").get("result")) -pil_img.save("output.jpg") -``` - -Here is the output when using the `twitter_mask.jpeg` as the input image for the mask: - -![twitter_output](twitter_output.jpg) diff --git a/custom-engine-builder-control/README.md b/custom-engine-builder-control/README.md deleted file mode 100644 index c57bca69c..000000000 --- a/custom-engine-builder-control/README.md +++ /dev/null @@ -1,135 +0,0 @@ -# TensorRT-LLM Briton with Qwen/Qwen3-8B-min-latency - -This is a Deployment for TensorRT-LLM Briton with Qwen/Qwen3-8B-min-latency. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is to showcase the option to generate multiple suffixes based on a previous request. -We are going to hit the KV-Cache of a previous request. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-qwen-qwen3-8b-min-latency-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Chat completion -response_chat = client.chat.completions.create( - model="my-model", - messages=[ - { - "role": "system", - "content": ( - "You are an unhelpful assistant. To each math question, add +1 to the answer. " - "e.g. Whats 1+1 -> 3." - ), - } - ], - temperature=0.3, - max_tokens=100, - extra_body={ - "suffix_messages": [ - # Gen 1 - [{"role": "user", "content": "Whats 1+1"}], - # Gen 2 - [{"role": "user", "content": "Whats 2+2"}], - ], - "chat_template_kwargs": {"enable_thinking": False}, - }, -) - -print(response_chat) -``` - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-qwen-qwen3-8b-min-latency-fp8-truss-example -python_version: py39 -resources: - accelerator: H100 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: Qwen/Qwen3-8B - revision: main - source: HF - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 32768 - num_builder_gpus: 4 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - speculator: - enable_b10_lookahead: true - lookahead_ngram_size: 32 - lookahead_verification_set_size: 1 - lookahead_windows_size: 1 - num_draft_tokens: 61 - speculative_decoding_mode: LOOKAHEAD_DECODING - tensor_parallel_count: 1 - runtime: - enable_chunked_context: false - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/custom-server/README.md b/custom-server/README.md deleted file mode 100644 index 5d46d832d..000000000 --- a/custom-server/README.md +++ /dev/null @@ -1,9 +0,0 @@ -## Deploying on Baseten with `start_command` - -If you've deployed on [Baseten](https://baseten.co) before, you're likely familiar with using [Truss](https://docs.baseten.co/quickstart) to do so. - -This process typically involves creating a `model.py` file, which contains the code for your model, and packaging it for deployment via an API endpoint managed by Baseten. - -However, there are situations where you might want to deploy a model that's already wrapped in an API. A common example is the `vLLM` OpenAI-Compatible Server, which provides its own HTTP endpoint. Another scenario is the "bring your own image" approach, where you have a Docker image that can handle HTTP requests directly. - -In these cases, setting up an HTTP endpoint through Truss or writing a `model.py` file can introduce unnecessary overhead. To streamline this, we've introduced the `docker_server.start_command` option, allowing you to specify an alternative `docker start` command, avoiding the need for additional setup. diff --git a/custom-server/infinity-embedding-server/README.md b/custom-server/infinity-embedding-server/README.md deleted file mode 100644 index afedc3615..000000000 --- a/custom-server/infinity-embedding-server/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Infinity Embedding Server Truss - -This is a [Truss](https://truss.baseten.co/) to deploy [infinity embedding server](https://github.com/michaelfeil/infinity), a high-throughput, low-latency REST API server for serving vector embeddings. - -## Deployment - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. [Required for gated model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd custom-server/infinity-embedding-server -``` - -With `infinity-embedding-server` as your working directory, you can deploy the model with the following command, paste your Baseten API key if prompted. - -```sh -truss push --publish --trusted -``` - -## Call your model - -### curl - -```bash -curl -X POST https://model-xxx.api.baseten.co/development/predict \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string"}' -``` - -### request python library - -```python -import requests - -resp = requests.post( - "https://model-xxx.api.baseten.co/development/predict", - headers={"Authorization": "Api-Key YOUR_API_KEY"}, - json={"input": "text string"}, -) - -print(resp.json()) -``` - -### openai python SDK - -```python -import os -from openai import OpenAI - -client = OpenAI( - api_key=os.environ["YOUR_API_KEY"], - base_url="https://bridge.baseten.co/v1/direct" -) - -model_id = "xxx" -deployment_id = "xxx" - -response = client.embeddings.create( - input="text string", - model="BAAI/bge-small-en-v1.5", - extra_body={ - "baseten": { - "model_id": model_id, - "deployment_id": deployment_id - } - } -) - -print(response.data[0].embedding) -``` - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/custom-server/pixtral-12b/README.md b/custom-server/pixtral-12b/README.md deleted file mode 100644 index acd17862f..000000000 --- a/custom-server/pixtral-12b/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Pixtral 12B Truss - -This is a [Truss](https://truss.baseten.co/) for Pixtral 12B. Pixtral is a 12B parameter language model released by [Mistral AI](https://mistral.ai/) and is a multimodal (text + vision) LLM. This Truss bypasses the need for writing a `model.py` and instead runs `vllm serve` directly at startup and uses the HTTP endpoint provided by the `vLLM` OpenAI Compatible Server to directly serve requests. This README will walk you through how to deploy this Truss on Baseten to get your own instance of it. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd custom-server/pixtral-12b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Accept the terms of service of the Pixtral model [here](https://huggingface.co/mistralai/Pixtral-12B-2409). -4. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -5. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. Note that you will *not* be able to successfully deploy Pixtral without doing this. - -With `pixtral-12b` as your working directory, you can deploy the model with: - -```sh -truss push --publish --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -You need one A100 to run Pixtral at `bf16`. - -## Pixtral 12B API documentation - -This section provides an overview of the Pixtral 12B API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The `predict` route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __messages__: The input in OpenAI API format (see below for examples) -- __stream__ (optional, default=False): A boolean determining whether the model should stream a response back. When `True`, the API returns generated text as it becomes available. -- __max_tokens__ (optional, default=512): Maximum number of tokens to generate -- __temperature__ (optional, default=0.7): A float between 0 and 1. Higher values means the generated output is more random while lower values means the generated output is more determenistic - -## Example usage - -```sh -truss predict -d '{"model": "pixtral", "messages": [{"role": "user", "content": "Tell me about yourself"}]}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "model": "pixtral", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What type of animal is this? Answer in French only" - }, - { - "type": "image_url", - "image_url": { - "url": "https://vetmed.illinois.edu/wp-content/uploads/2021/04/pc-keller-hedgehog.jpg" - } - } - ] - } - ], - "stream": true, - "max_tokens": 64, - "temperature": 0.2 - }' --no-buffer -``` diff --git a/custom-server/ultravox-0.4/README.md b/custom-server/ultravox-0.4/README.md deleted file mode 100644 index 363e05e4d..000000000 --- a/custom-server/ultravox-0.4/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Ultravox v0.4 vLLM Truss - -Ultravox is a multimodal model that can consume both speech and text as input, generating output text as usual. - -This is a [Truss](https://truss.baseten.co/) for Ultravox using the vLLM OpenAI Compatible server. This Truss bypasses the need for writing a `model.py` and instead runs `vllm serve` directly at startup and uses the HTTP endpoint provided by `vLLM` OpenAI Compatible Server to directly serve requests. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd custom-server/ultravox-0.4 -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. Note that you will *not* be able to successfully deploy Ultravox without doing this. - -With `ultravox-0.4` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -model_id = "jwdp26kw" # Replace with your model ID - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -response = client.chat.completions.create( - model="ultravox", # Replace with your model name - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is Lydia like?" - }, - { - "type": "audio_url", - "audio_url": {"url": "https://baseten-public.s3.us-west-2.amazonaws.com/fred-audio-tests/real.mp3"} - } - ] - } - ], - stream=True -) - -for chunk in response: - content = chunk.choices[0].delta.content - print(content, end="", flush=True) -``` - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/custom-server/ultravox-0.5-8b/README.md b/custom-server/ultravox-0.5-8b/README.md deleted file mode 100644 index 1acce3e22..000000000 --- a/custom-server/ultravox-0.5-8b/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Ultravox v0.5 8B vLLM Truss - -Ultravox is a multimodal model that can consume both speech and text as input, generating output text as usual. - -This is a [Truss](https://truss.baseten.co/) for Ultravox using the vLLM OpenAI Compatible server. This Truss bypasses the need for writing a `model.py` and instead runs `vllm serve` directly at startup and uses the HTTP endpoint provided by `vLLM` OpenAI Compatible Server to directly serve requests. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd custom-server/ultravox-0.5-8b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. Note that you will *not* be able to successfully deploy Ultravox without doing this. - -With `ultravox-0.5-8b` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -model_id = "jwdp26kw" # Replace with your model ID - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -response = client.chat.completions.create( - model="ultravox", # Replace with your model name - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is Lydia like?" - }, - { - "type": "audio_url", - "audio_url": {"url": "https://baseten-public.s3.us-west-2.amazonaws.com/fred-audio-tests/real.mp3"} - } - ] - } - ], - stream=True -) - -for chunk in response: - content = chunk.choices[0].delta.content - print(content, end="", flush=True) -``` - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/custom-server/ultravox-0.6-70b/README.md b/custom-server/ultravox-0.6-70b/README.md deleted file mode 100644 index 815b89ec9..000000000 --- a/custom-server/ultravox-0.6-70b/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Ultravox v0.6 70B vLLM Truss - -Ultravox is a multimodal model that can consume both speech and text as input, generating output text as usual. - -This is a [Truss](https://truss.baseten.co/) for Ultravox using the vLLM OpenAI Compatible server. This Truss bypasses the need for writing a `model.py` and instead runs `vllm serve` directly at startup and uses the HTTP endpoint provided by `vLLM` OpenAI Compatible Server to directly serve requests. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd custom-server/ultravox-0.6-70b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. Note that you will *not* be able to successfully deploy Ultravox without doing this. - -With `ultravox-0.6-70b` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -model_id = "jwdp26kw" # Replace with your model ID - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -response = client.chat.completions.create( - model="ultravox", # Replace with your model name - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is Lydia like?" - }, - { - "type": "audio_url", - "audio_url": {"url": "https://baseten-public.s3.us-west-2.amazonaws.com/fred-audio-tests/real.mp3"} - } - ] - } - ], - stream=True -) - -for chunk in response: - content = chunk.choices[0].delta.content - print(content, end="", flush=True) -``` - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/custom-server/voxtral-mini-3b-2507/README.md b/custom-server/voxtral-mini-3b-2507/README.md deleted file mode 100644 index 1da9e95f5..000000000 --- a/custom-server/voxtral-mini-3b-2507/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# Voxtral Mini 3B 2507 vLLM Truss - -Voxtral Mini is an enhancement of Ministral 3B, incorporating state-of-the-art audio input capabilities while retaining best-in-class text performance. It excels at speech transcription, translation and audio understanding. - -This is a [Truss](https://truss.baseten.co/) for Voxtral Mini using the vLLM OpenAI Compatible server. This Truss bypasses the need for writing a `model.py` and instead runs `vllm serve` directly at startup and uses the HTTP endpoint provided by `vLLM` OpenAI Compatible Server to directly serve requests. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd custom-server/voxtral-mini-3b-2507 -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. Note that you will _not_ be able to successfully deploy Voxtral Mini without doing this. - -With `voxtral-mini-3b-2507` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from mistral_common.protocol.instruct.messages import ( - TextChunk, - AudioChunk, - UserMessage, - AssistantMessage, - RawAudio, -) -from mistral_common.audio import Audio -from huggingface_hub import hf_hub_download - -from openai import OpenAI - -model_id = "12345678" - -client = OpenAI( - api_key="YOUR_API_KEY", - base_url=f"https://model-{model_id}.api.baseten.co/{deploy_env}/sync/v1" -) - -models = client.models.list() -model = models.data[0].id - -obama_file = hf_hub_download( - "patrickvonplaten/audio_samples", "obama.mp3", repo_type="dataset" -) -bcn_file = hf_hub_download( - "patrickvonplaten/audio_samples", "bcn_weather.mp3", repo_type="dataset" -) - -def file_to_chunk(file: str) -> AudioChunk: - audio = Audio.from_file(file, strict=False) - return AudioChunk.from_audio(audio) - -text_chunk = TextChunk( - text="Which speaker is more inspiring? Why? How are they different from each other?" -) -user_msg = UserMessage( - content=[file_to_chunk(obama_file), file_to_chunk(bcn_file), text_chunk] -).to_openai() - -print(30 * "=" + "USER 1" + 30 * "=") -print(text_chunk.text) -print("\n\n") - -response = client.chat.completions.create( - model=model, - messages=[user_msg], - temperature=0.2, - top_p=0.95, -) -content = response.choices[0].message.content - -print(30 * "=" + "BOT 1" + 30 * "=") -print(content) -print("\n\n") - -messages = [ - user_msg, - AssistantMessage(content=content).to_openai(), - UserMessage( - content="Ok, now please summarize the content of the first audio." - ).to_openai(), -] -print(30 * "=" + "USER 2" + 30 * "=") -print(messages[-1]["content"]) -print("\n\n") - -response = client.chat.completions.create( - model=model, - messages=messages, - temperature=0.2, - top_p=0.95, -) -content = response.choices[0].message.content -print(30 * "=" + "BOT 2" + 30 * "=") -print(content) - -``` - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/custom-server/voxtral-small-24b-2507/README.md b/custom-server/voxtral-small-24b-2507/README.md deleted file mode 100644 index f2ddd4952..000000000 --- a/custom-server/voxtral-small-24b-2507/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# Voxtral Small 24B 2507 vLLM Truss - -Voxtral Small is an enhancement of Mistral Small 3, incorporating state-of-the-art audio input capabilities while retaining best-in-class text performance. It excels at speech transcription, translation and audio understanding. - -This is a [Truss](https://truss.baseten.co/) for Voxtral Small using the vLLM OpenAI Compatible server. This Truss bypasses the need for writing a `model.py` and instead runs `vllm serve` directly at startup and uses the HTTP endpoint provided by `vLLM` OpenAI Compatible Server to directly serve requests. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd custom-server/voxtral-small-24b-2507 -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -4. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. Note that you will _not_ be able to successfully deploy Voxtral Small without doing this. - -With `voxtral-small-24b-2507` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) without the need for a `model.py` through the `docker_server.start_command` option. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from mistral_common.protocol.instruct.messages import ( - TextChunk, - AudioChunk, - UserMessage, - AssistantMessage, - RawAudio, -) -from mistral_common.audio import Audio -from huggingface_hub import hf_hub_download - -from openai import OpenAI - -model_id = "12345678" - -client = OpenAI( - api_key="YOUR_API_KEY", - base_url=f"https://model-{model_id}.api.baseten.co/{deploy_env}/sync/v1" -) - -models = client.models.list() -model = models.data[0].id - -obama_file = hf_hub_download( - "patrickvonplaten/audio_samples", "obama.mp3", repo_type="dataset" -) -bcn_file = hf_hub_download( - "patrickvonplaten/audio_samples", "bcn_weather.mp3", repo_type="dataset" -) - -def file_to_chunk(file: str) -> AudioChunk: - audio = Audio.from_file(file, strict=False) - return AudioChunk.from_audio(audio) - -text_chunk = TextChunk( - text="Which speaker is more inspiring? Why? How are they different from each other? Answer in French." -) -user_msg = UserMessage( - content=[file_to_chunk(obama_file), file_to_chunk(bcn_file), text_chunk] -).to_openai() - -print(30 * "=" + "USER 1" + 30 * "=") -print(text_chunk.text) -print("\n\n") - -response = client.chat.completions.create( - model=model, - messages=[user_msg], - temperature=0.2, - top_p=0.95, -) -content = response.choices[0].message.content - -print(30 * "=" + "BOT 1" + 30 * "=") -print(content) -print("\n\n") - -messages = [ - user_msg, - AssistantMessage(content=content).to_openai(), - UserMessage( - content="Ok, now please summarize the content of the first audio." - ).to_openai(), -] -print(30 * "=" + "USER 2" + 30 * "=") -print(messages[-1]["content"]) -print("\n\n") - -response = client.chat.completions.create( - model=model, - messages=messages, - temperature=0.2, - top_p=0.95, -) -content = response.choices[0].message.content -print(30 * "=" + "BOT 2" + 30 * "=") -print(content) - -``` - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/deepfloyd-xl/README.md b/deepfloyd-xl/README.md deleted file mode 100644 index 0eda7c075..000000000 --- a/deepfloyd-xl/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# DeepFloyd XL Truss - -This is a [Truss](https://truss.baseten.co/) for DeepFloyd-IF. DeepFloyd-IF is a pixel-based text-to-image triple-cascaded diffusion model that can generate pictures and sets a new state-of-the-art for photorealism and language understanding. The result is a highly efficient model that outperforms current state-of-the-art models, achieving a zero-shot FID-30K score of 6.66 on the COCO dataset. - -Model details: - -- Developed by: DeepFloyd, StabilityAI -- Model type: pixel-based text-to-image cascaded diffusion model -- Cascade Stage: I -- Num Parameters: 4.3B -- Language(s): primarily English and, to a lesser extent, other Romance languages -- License: [DeepFloyd IF License Agreement](https://huggingface.co/spaces/DeepFloyd/deepfloyd-if-license) -- Model Description: DeepFloyd-IF is modular composed of frozen text mode and three pixel cascaded diffusion modules, each designed to generate images of increasing resolution: 64x64, 256x256, and 1024x1024. All stages of the model utilize a frozen text encoder based on the T5 transformer to extract text embeddings, which are then fed into a UNet architecture enhanced with cross-attention and attention-pooling - -Before deploying this model, you'll need to: - -1. Accept the terms of service of the Deepfloyd XL model [here](https://huggingface.co/DeepFloyd/IF-I-XL-v1.0). -2. Retrieve your Huggingface token from the [settings](https://huggingface.co/settings/tokens). -3. Set your Huggingface token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. - -## Deploying DeepFloyd XL - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd deepfloyd-xl-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `deepfloyd-xl-truss` as your working directory, you can deploy the model with: - -```sh -truss push --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## DeepFloyd API documentation - -### Input - -This deployment of DeepFloyd takes a dictionary as input, which requires the following key: - -- `prompt` - the prompt for image generation - -It also supports a number of other parameters detailed in [this blog post](https://huggingface.co/blog/if). - -### Output - -The result will be a dictionary containing: - -- `status` - either `success` or `failed` -- `data` - list of base 64 encoded images -- `message` - will contain details in the case of errors - -```json -{ - "status": "success", - "data": ["/9j/4AAQSkZJRgABAQAAAQABAA...."], - "message": null -} -``` - -## Example usage - -```sh -truss predict -d '{"prompt": "man on moon"}' -``` - -You can also invoke it via cURL: - -```sh -curl -X POST https://app.baseten.co/models/EqwKvqa/predict \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{"prompt": "man on moon"}' -``` diff --git a/deepseek-ocr/README.md b/deepseek-ocr/README.md deleted file mode 100644 index 07aaa92f2..000000000 --- a/deepseek-ocr/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# DeepSeek-OCR Truss Model - -This is a Truss deployment of the DeepSeek OCR model for optical character recognition using vLLM engine on Baseten served on a H100_40G. The model excels at reading handwritten text, documents, and complex layouts with bounding box detection. - -DeepSeek-OCR processes 200k+ pages/day on a single GPU or 33M pages/day on 20 nodes. It requires 10x fewer visual tokens than text tokens, which means OCR compresses information 10x more efficiently than the text, with decoding precision of 97%. This makes it an excellent model for generating training data as well as potential tasks that involve long-context windows and memory. - -## Quick Start - -### 1. Deploy to Baseten - -```bash -# Set your Baseten API key -export BASETEN_API_KEY="your_api_key_here" - -# Clone this repo and cd into this folder -git clone https://github.com/basetenlabs/truss-examples.git -cd truss-examples/deepseek-ocr - -# Deploy the model -truss push --publish -# This assumes you have truss installed, if not follow the instructions here: -# https://docs.baseten.co/development/model/build-your-first-model -``` - -### 2. Test with Sample Image - -Replace line 16 `ENDPOINT_URL` in `test_document_ocr.py` with your specific deployment URL. -```bash -# Run the test script with Bad-Handwriting.png -python test_document_ocr.py -``` - -This will: -- Load `Bad-Handwriting.png` (a challenging handwriting sample) -- Test 5 different OCR prompts -- Generate visualizations with bounding boxes -- Save results as `visualization_*.png` files - -### 3. Project Structure - -``` -deepseek-ocr/ -β”œβ”€β”€ config.yaml # Truss configuration -β”œβ”€β”€ model/ -β”‚ β”œβ”€β”€ __init__.py -β”‚ └── model.py # Main model implementation -β”œβ”€β”€ README.md # Documentation -β”œβ”€β”€ test_document_ocr.py # Working test script -β”œβ”€β”€ visualizer.py # Bounding box visualization -β”œβ”€β”€ Bad-Handwriting.png # Test image -└── visualization_*.png # Generated outputs -``` - -## Model Information - -- **Model**: DeepSeek OCR v1 -- **Framework**: vLLM + PyTorch + Transformers -- **GPU**: H100_40GB recommended -- **Memory**: 16GB RAM -- **Engine**: AsyncLLMEngine with custom DeepseekOCRForCausalLM - -## Usage - -### Using the Test Script - -The `test_document_ocr.py` script is the main testing interface and provides a complete example of how to use the model: - -```python -# The script tests these prompts: -prompts = [ - "\n<|grounding|>Convert the document to markdown.", - "\n<|grounding|>OCR this image.", - "\nFree OCR.", - "\nParse the figure.", - "\nDescribe this image in detail.", -] -``` - -**Recommended**: Use `\n<|grounding|>Convert the document to markdown.` for best results with bounding boxes. - -For detailed API documentation and examples, see the `test_document_ocr.py` script. diff --git a/deepseek/engine-deepseek-r1-distill-llama-70b/README.md b/deepseek/engine-deepseek-r1-distill-llama-70b/README.md deleted file mode 100644 index bf3784d87..000000000 --- a/deepseek/engine-deepseek-r1-distill-llama-70b/README.md +++ /dev/null @@ -1 +0,0 @@ -# DeepSeek-R1 Distill Llama 70B diff --git a/deepseek/engine-deepseek-r1-distill-llama-8b/README.md b/deepseek/engine-deepseek-r1-distill-llama-8b/README.md deleted file mode 100644 index c55238dc2..000000000 --- a/deepseek/engine-deepseek-r1-distill-llama-8b/README.md +++ /dev/null @@ -1 +0,0 @@ -# DeepSeek-R1 Distill Llama 8B diff --git a/deepseek/engine-deepseek-r1-distill-qwen-14b/README.md b/deepseek/engine-deepseek-r1-distill-qwen-14b/README.md deleted file mode 100644 index abd78652a..000000000 --- a/deepseek/engine-deepseek-r1-distill-qwen-14b/README.md +++ /dev/null @@ -1 +0,0 @@ -# DeepSeek-R1 Distill Qwen 14B diff --git a/deepseek/engine-deepseek-r1-distill-qwen-32b/README.md b/deepseek/engine-deepseek-r1-distill-qwen-32b/README.md deleted file mode 100644 index e051dfc88..000000000 --- a/deepseek/engine-deepseek-r1-distill-qwen-32b/README.md +++ /dev/null @@ -1 +0,0 @@ -# DeepSeek-R1 Distill Qwen 32B diff --git a/deepseek/engine-deepseek-r1-distill-qwen-7b/README.md b/deepseek/engine-deepseek-r1-distill-qwen-7b/README.md deleted file mode 100644 index 97210c5df..000000000 --- a/deepseek/engine-deepseek-r1-distill-qwen-7b/README.md +++ /dev/null @@ -1 +0,0 @@ -# DeepSeek-R1 Distill Qwen 7B diff --git a/dis-segmentation/README.md b/dis-segmentation/README.md deleted file mode 100644 index f6b09639f..000000000 --- a/dis-segmentation/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# DIS Segmentation Truss - -This is a [Truss](https://truss.baseten.co/welcome) for [DIS](https://github.com/xuebinqin/DIS/tree/main) - -This model can be used to remove backgrounds from an image or create a segmentation mask for an object. - -This model takes an input image and creates two output images: -1. This first output image is the original image with the background removed -2. The second output image contains the mask of the object in the foreground - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd dis-segmentation -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `dis-segmentation` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### API route: `predict` - -The model only has one input: - -- `input_image` (required): An image represented as a base74 string. - -The output of the model contains two images in the form of a base64 string. -Example model output: -```json -{ - "img_without_bg": "Base64 string image", - "image_mask": "Base64 string image" -} -``` - -## Example usage - -You can invoke the model in Python like so: - -``` python -import os -import json -import base64 -import requests - -# Set essential values -model_id = "" -baseten_api_key = "" - -BASE64_PREAMBLE = "data:image/png;base64," - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/development/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json={ - "input_image": pil_to_b64(Image.open("path/to/image.png")) - } -) -# Get output image -res = res.json() -background_removed = b64_to_pil(res.get("img_without_bg")) -mask = b64_to_pil(res.get("image_mask")) -background_removed.save("image_without_background.png") -mask.save("image_mask.png") -``` - -Here is an example of the outputs given the following input. - -Input image: - -![input image](https://github.com/basetenlabs/truss-examples/assets/15642666/a0491a6f-795b-4e2a-aa66-8830f4fb86b3) - -Image without background: - -![removed-bg](https://github.com/basetenlabs/truss-examples/assets/15642666/a94ceba8-9a62-4e38-9fbe-f6f9382c8086) - -Image mask: - -![mask](https://github.com/basetenlabs/truss-examples/assets/15642666/14bcaea1-2136-4cdb-b896-bbed9e99fa89) diff --git a/embeddings/README.md b/embeddings/README.md new file mode 100644 index 000000000..727683129 --- /dev/null +++ b/embeddings/README.md @@ -0,0 +1,18 @@ +# Embeddings + +Truss configurations for text embedding models, rerankers, and classifiers. Includes both TensorRT-optimized and HuggingFace TEI-based deployments covering a broad set of embedding providers. + +| Directory | Models | Description | +|-----------|--------|-------------| +| [bei](bei/) | 50 | Baseten Embeddings Infrastructure -- TensorRT-optimized embedding, reranking, and classification models from providers including BGE, GTE, Nomic, Jina, Qwen 3, Snowflake, and more | +| [tei](tei/) | 15 | HuggingFace Text Embeddings Inference server configurations for embedding and reranking models | +| [clip](clip/) | 1 | OpenAI CLIP model for image and text embeddings | +| [text-embeddings-inference](text-embeddings-inference/) | 1 | Standalone Text Embeddings Inference server configuration | + +## Deploying + +Each embedding model can be deployed to Baseten with: + +```bash +truss push +``` diff --git a/embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding/README.md b/embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding/README.md new file mode 100644 index 000000000..db6d5f697 --- /dev/null +++ b/embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding/README.md @@ -0,0 +1,30 @@ +# Alibaba-NLP GTE ModernBERT Base Embedding + +Deploy [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Alibaba-NLP/gte-modernbert-base"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding/config.yaml b/embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding/config.yaml rename to embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding/config.yaml index 08fc103c6..b1fa8af5a 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-modernbert-base-embedding/config.yaml +++ b/embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-modernbert-base embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md b/embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md new file mode 100644 index 000000000..642fb2cda --- /dev/null +++ b/embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md @@ -0,0 +1,30 @@ +# Alibaba-NLP GTE Qwen2 1.5B Instruct Embedding + +Deploy [Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml b/embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml rename to embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml index 06d001826..339d0e512 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml +++ b/embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-Qwen2-1.5B-instruct embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md b/embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md new file mode 100644 index 000000000..b729aeb90 --- /dev/null +++ b/embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md @@ -0,0 +1,30 @@ +# Alibaba-NLP GTE Qwen2 7B Instruct Embedding + +Deploy [Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Alibaba-NLP/gte-Qwen2-7B-instruct"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml b/embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml rename to embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml index f95eb18ba..f8d611ee0 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml +++ b/embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-Qwen2-7B-instruct embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base/README.md b/embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base/README.md new file mode 100644 index 000000000..4bb975fba --- /dev/null +++ b/embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base/README.md @@ -0,0 +1,30 @@ +# Alibaba-NLP GTE Reranker ModernBERT Base + +Deploy [Alibaba-NLP/gte-reranker-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-reranker-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base/config.yaml b/embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base/config.yaml similarity index 85% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base/config.yaml rename to embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base/config.yaml index c0afa0b0d..8f935dea3 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-alibaba-nlp-gte-reranker-modernbert-base/config.yaml +++ b/embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-reranker-modernbert-base reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: query: What is Baseten? raw_scores: true diff --git a/embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md b/embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md new file mode 100644 index 000000000..9d5df943f --- /dev/null +++ b/embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8/README.md @@ -0,0 +1,43 @@ +# AllenAI Llama 3.1 Tulu 3 8B Reward Model + +Deploy [allenai/Llama-3.1-Tulu-3-8B-RM](https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B-RM) for classification using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [allenai/Llama-3.1-Tulu-3-8B-RM](https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B-RM) | +| Task | Classification | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "inputs": [ + [ + "Baseten is a fast inference provider" + ], + [ + "Classify this separately." + ] + ], + "raw_scores": true, + "truncate": true, + "truncation_direction": "Right" +}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8/config.yaml b/embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8/config.yaml rename to embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8/config.yaml index 7b45b33c0..ab5866f63 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-allenai-llama-3.1-tulu-3-8b-reward-model-fp8/config.yaml +++ b/embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "allenai/Llama-3.1-Tulu-3-8B-RM classifier model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/baai-bge-en-icl-embedding-fp8/README.md b/embeddings/bei/baai-bge-en-icl-embedding-fp8/README.md new file mode 100644 index 000000000..b3cb98355 --- /dev/null +++ b/embeddings/bei/baai-bge-en-icl-embedding-fp8/README.md @@ -0,0 +1,31 @@ +# BAAI BGE EN ICL Embedding + +Deploy [BAAI/bge-en-icl](https://huggingface.co/BAAI/bge-en-icl) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-en-icl](https://huggingface.co/BAAI/bge-en-icl) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "BAAI/bge-en-icl"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8/config.yaml b/embeddings/bei/baai-bge-en-icl-embedding-fp8/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8/config.yaml rename to embeddings/bei/baai-bge-en-icl-embedding-fp8/config.yaml index f5919a8f7..77b9c39b6 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-en-icl-embedding-fp8/config.yaml +++ b/embeddings/bei/baai-bge-en-icl-embedding-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-en-icl embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/baai-bge-large-en-v1.5-embedding/README.md b/embeddings/bei/baai-bge-large-en-v1.5-embedding/README.md new file mode 100644 index 000000000..727e6c107 --- /dev/null +++ b/embeddings/bei/baai-bge-large-en-v1.5-embedding/README.md @@ -0,0 +1,30 @@ +# BAAI BGE Large EN v1.5 Embedding + +Deploy [BAAI/bge-large-en-v1.5](https://huggingface.co/BAAI/bge-large-en-v1.5) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-large-en-v1.5](https://huggingface.co/BAAI/bge-large-en-v1.5) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "BAAI/bge-large-en-v1.5"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding/config.yaml b/embeddings/bei/baai-bge-large-en-v1.5-embedding/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding/config.yaml rename to embeddings/bei/baai-bge-large-en-v1.5-embedding/config.yaml index d61622687..4336bcf45 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-large-en-v1.5-embedding/config.yaml +++ b/embeddings/bei/baai-bge-large-en-v1.5-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-large-en-v1.5 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/baai-bge-m3-embedding-dense/README.md b/embeddings/bei/baai-bge-m3-embedding-dense/README.md new file mode 100644 index 000000000..7b197b215 --- /dev/null +++ b/embeddings/bei/baai-bge-m3-embedding-dense/README.md @@ -0,0 +1,30 @@ +# BAAI BGE M3 Embedding Dense + +Deploy [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "BAAI/bge-m3"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense/config.yaml b/embeddings/bei/baai-bge-m3-embedding-dense/config.yaml similarity index 85% rename from 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense/config.yaml rename to embeddings/bei/baai-bge-m3-embedding-dense/config.yaml index 80ce00928..10edada5d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-m3-embedding-dense/config.yaml +++ b/embeddings/bei/baai-bge-m3-embedding-dense/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-m3 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding/README.md b/embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding/README.md new file mode 100644 index 000000000..c7fda032e --- /dev/null +++ b/embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding/README.md @@ -0,0 +1,30 @@ +# BAAI BGE Multilingual Gemma2 Embedding + +Deploy [BAAI/bge-multilingual-gemma2](https://huggingface.co/BAAI/bge-multilingual-gemma2) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-multilingual-gemma2](https://huggingface.co/BAAI/bge-multilingual-gemma2) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "BAAI/bge-multilingual-gemma2"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding/config.yaml b/embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding/config.yaml rename to embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding/config.yaml index 6ec3d19f2..b13aef4a3 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-multilingual-gemma2-multilingual-embedding/config.yaml +++ b/embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-multilingual-gemma2 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large/README.md b/embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large/README.md new file mode 100644 index 000000000..fad7cb7b0 --- /dev/null +++ b/embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large/README.md @@ -0,0 +1,30 @@ +# BAAI BGE Reranker Large + +Deploy [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large/config.yaml b/embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large/config.yaml rename to embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large/config.yaml index 07cee6c55..8294976a6 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-baai-bge-reranker-large/config.yaml +++ b/embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-reranker-large reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: query: What is Baseten? raw_scores: true diff --git a/embeddings/bei/baai-bge-reranker-large/README.md b/embeddings/bei/baai-bge-reranker-large/README.md new file mode 100644 index 000000000..d0db506d5 --- /dev/null +++ b/embeddings/bei/baai-bge-reranker-large/README.md @@ -0,0 +1,30 @@ +# BAAI BGE Reranker Large + +Deploy [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large/config.yaml b/embeddings/bei/baai-bge-reranker-large/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large/config.yaml rename to embeddings/bei/baai-bge-reranker-large/config.yaml index 0e0ef388d..b02f57fda 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-large/config.yaml +++ b/embeddings/bei/baai-bge-reranker-large/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-reranker-large reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: query: What is Baseten? raw_scores: true diff --git a/embeddings/bei/baai-bge-reranker-v2-m3-multilingual/README.md b/embeddings/bei/baai-bge-reranker-v2-m3-multilingual/README.md new file mode 100644 index 000000000..7740d4f55 --- /dev/null +++ b/embeddings/bei/baai-bge-reranker-v2-m3-multilingual/README.md @@ -0,0 +1,30 @@ +# BAAI BGE Reranker v2 M3 + +Deploy [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual/config.yaml b/embeddings/bei/baai-bge-reranker-v2-m3-multilingual/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual/config.yaml rename to embeddings/bei/baai-bge-reranker-v2-m3-multilingual/config.yaml index 78b8a92b6..d8fca2e5e 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baai-bge-reranker-v2-m3-multilingual/config.yaml +++ b/embeddings/bei/baai-bge-reranker-v2-m3-multilingual/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-reranker-v2-m3 reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: query: What is Baseten? raw_scores: true diff --git a/embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md b/embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md new file mode 100644 index 000000000..e453ca27a --- /dev/null +++ b/embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/README.md @@ -0,0 +1,43 @@ +# Baseten Example Meta Llama 3 70B InstructForSequenceClassification + +Deploy [baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification](https://huggingface.co/baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification) for classification using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification](https://huggingface.co/baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification) | +| Task | Classification | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "inputs": [ + [ + "Baseten is a fast inference provider" + ], + [ + "Classify this separately." + ] + ], + "raw_scores": true, + "truncate": true, + "truncation_direction": "Right" +}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/config.yaml b/embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/config.yaml rename to embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/config.yaml index 365fb6a67..97146ab5d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/config.yaml +++ b/embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "baseten/example-Meta-Llama-3-70B-InstructForSequenceClassification classifier model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8/README.md b/embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8/README.md new file mode 100644 index 000000000..6d17eb461 --- /dev/null +++ b/embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8/README.md @@ -0,0 +1,31 @@ +# Codefuse-AI F2LLM 4B Embedding + +Deploy [codefuse-ai/F2LLM-4B](https://huggingface.co/codefuse-ai/F2LLM-4B) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [codefuse-ai/F2LLM-4B](https://huggingface.co/codefuse-ai/F2LLM-4B) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "codefuse-ai/F2LLM-4B"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8/config.yaml b/embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8/config.yaml similarity index 85% rename from 11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8/config.yaml rename to embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8/config.yaml index 151d24a5b..a19bf13b9 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-codefuse-ai-f2llm-4b-embedding-fp8/config.yaml +++ b/embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "codefuse-ai/F2LLM-4B embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/google-embeddinggemma-300m/README.md b/embeddings/bei/google-embeddinggemma-300m/README.md new file mode 100644 index 000000000..655480877 --- /dev/null +++ b/embeddings/bei/google-embeddinggemma-300m/README.md @@ -0,0 +1,30 @@ +# Google EmbeddingGemma 300M Embedding + +Deploy [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "google/embeddinggemma-300m"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m/config.yaml b/embeddings/bei/google-embeddinggemma-300m/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m/config.yaml rename to embeddings/bei/google-embeddinggemma-300m/config.yaml index f995b6e69..ae39fa87f 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-google-embeddinggemma-300m/config.yaml +++ b/embeddings/bei/google-embeddinggemma-300m/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "google/embeddinggemma-300m embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md b/embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md new file mode 100644 index 000000000..28c68545a --- /dev/null +++ b/embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8/README.md @@ -0,0 +1,31 @@ +# Intfloat E5 Mistral 7B Instruct Embedding + +Deploy [intfloat/e5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [intfloat/e5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "intfloat/e5-mistral-7b-instruct"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8/config.yaml b/embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8/config.yaml rename to embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8/config.yaml index 2bc0e145c..1b7e09d6a 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-intfloat-e5-mistral-7b-instruct-embedding-fp8/config.yaml +++ b/embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "intfloat/e5-mistral-7b-instruct embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/intfloat-multilingual-e5-large-instruct/README.md b/embeddings/bei/intfloat-multilingual-e5-large-instruct/README.md new file mode 100644 index 000000000..b1fd2d32a --- /dev/null +++ b/embeddings/bei/intfloat-multilingual-e5-large-instruct/README.md @@ -0,0 +1,30 @@ +# Intfloat Multilingual E5 Large Instruct Embedding + +Deploy [intfloat/multilingual-e5-large-instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [intfloat/multilingual-e5-large-instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "intfloat/multilingual-e5-large-instruct"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct/config.yaml b/embeddings/bei/intfloat-multilingual-e5-large-instruct/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct/config.yaml rename to embeddings/bei/intfloat-multilingual-e5-large-instruct/config.yaml index 5526ab547..21e4f4176 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-intfloat-multilingual-e5-large-instruct/config.yaml +++ b/embeddings/bei/intfloat-multilingual-e5-large-instruct/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "intfloat/multilingual-e5-large-instruct embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/jina-ai-jina-embeddings-v2-base-en/README.md b/embeddings/bei/jina-ai-jina-embeddings-v2-base-en/README.md new file mode 100644 index 000000000..9e2410619 --- /dev/null +++ b/embeddings/bei/jina-ai-jina-embeddings-v2-base-en/README.md @@ -0,0 +1,30 @@ +# Jina AI Jina Embeddings v2 Base EN + +Deploy [jinaai/jina-embeddings-v2-base-en](https://huggingface.co/jinaai/jina-embeddings-v2-base-en) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [jinaai/jina-embeddings-v2-base-en](https://huggingface.co/jinaai/jina-embeddings-v2-base-en) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "jinaai/jina-embeddings-v2-base-en"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en/config.yaml b/embeddings/bei/jina-ai-jina-embeddings-v2-base-en/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en/config.yaml rename to embeddings/bei/jina-ai-jina-embeddings-v2-base-en/config.yaml index 5d201f46c..1253f9d1d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jina-ai-jina-embeddings-v2-base-en/config.yaml +++ b/embeddings/bei/jina-ai-jina-embeddings-v2-base-en/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "jinaai/jina-embeddings-v2-base-en embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8/README.md b/embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8/README.md new file mode 100644 index 000000000..ef3e26f40 --- /dev/null +++ b/embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8/README.md @@ -0,0 +1,31 @@ +# Jina AI Jina Code Embeddings 0.5B + +Deploy [jinaai/jina-code-embeddings-0.5b](https://huggingface.co/jinaai/jina-code-embeddings-0.5b) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [jinaai/jina-code-embeddings-0.5b](https://huggingface.co/jinaai/jina-code-embeddings-0.5b) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "jinaai/jina-code-embeddings-0.5b"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8/config.yaml b/embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8/config.yaml rename to embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8/config.yaml index 3ec46322f..534d1f269 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-jinaai-jina-code-embeddings-0.5b-fp8/config.yaml +++ b/embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "jinaai/jina-code-embeddings-0.5b embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/jinaai-jina-embeddings-v2-base-code/README.md b/embeddings/bei/jinaai-jina-embeddings-v2-base-code/README.md new file mode 100644 index 000000000..1a4cdd7dc --- /dev/null +++ b/embeddings/bei/jinaai-jina-embeddings-v2-base-code/README.md @@ -0,0 +1,30 @@ +# Jina AI Jina Embeddings v2 Base Code + +Deploy [jinaai/jina-embeddings-v2-base-code](https://huggingface.co/jinaai/jina-embeddings-v2-base-code) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [jinaai/jina-embeddings-v2-base-code](https://huggingface.co/jinaai/jina-embeddings-v2-base-code) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "jinaai/jina-embeddings-v2-base-code"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code/config.yaml b/embeddings/bei/jinaai-jina-embeddings-v2-base-code/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code/config.yaml rename to embeddings/bei/jinaai-jina-embeddings-v2-base-code/config.yaml index c27ee0692..54bdd69c8 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-jinaai-jina-embeddings-v2-base-code/config.yaml +++ b/embeddings/bei/jinaai-jina-embeddings-v2-base-code/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "jinaai/jina-embeddings-v2-base-code embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md new file mode 100644 index 000000000..d57ec58b2 --- /dev/null +++ b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/README.md @@ -0,0 +1,30 @@ +# Mixedbread AI MxBAI Embed Large v1 Embedding + +Deploy [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "mixedbread-ai/mxbai-embed-large-v1"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml rename to embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml index 54a55f9fe..fc42152d7 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml +++ b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "mixedbread-ai/mxbai-embed-large-v1 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md new file mode 100644 index 000000000..d57ec58b2 --- /dev/null +++ b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md @@ -0,0 +1,30 @@ +# Mixedbread AI MxBAI Embed Large v1 Embedding + +Deploy [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "mixedbread-ai/mxbai-embed-large-v1"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml rename to embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml index a1ac58726..298732476 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml +++ b/embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "mixedbread-ai/mxbai-embed-large-v1 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md b/embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md new file mode 100644 index 000000000..b19dda6f1 --- /dev/null +++ b/embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/README.md @@ -0,0 +1,31 @@ +# MxBAI Rerank Base v2 Reranker + +Deploy [michaelfeil/mxbai-rerank-base-v2-seq](https://huggingface.co/michaelfeil/mxbai-rerank-base-v2-seq) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/mxbai-rerank-base-v2-seq](https://huggingface.co/michaelfeil/mxbai-rerank-base-v2-seq) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/config.yaml b/embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/config.yaml rename to embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/config.yaml index af2ccf720..cf3b77360 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/config.yaml +++ b/embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/mxbai-rerank-base-v2-seq reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md b/embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md new file mode 100644 index 000000000..3b5ca02ca --- /dev/null +++ b/embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/README.md @@ -0,0 +1,31 @@ +# MxBAI Rerank Large v2 Reranker + +Deploy [michaelfeil/mxbai-rerank-large-v2-seq](https://huggingface.co/michaelfeil/mxbai-rerank-large-v2-seq) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/mxbai-rerank-large-v2-seq](https://huggingface.co/michaelfeil/mxbai-rerank-large-v2-seq) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/config.yaml b/embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/config.yaml rename to embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/config.yaml index 74b3bd80c..e5739a2dc 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/config.yaml +++ b/embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/mxbai-rerank-large-v2-seq reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/ncbi-medcpt-cross-encoder-reranker/README.md b/embeddings/bei/ncbi-medcpt-cross-encoder-reranker/README.md new file mode 100644 index 000000000..ffbf0e99b --- /dev/null +++ b/embeddings/bei/ncbi-medcpt-cross-encoder-reranker/README.md @@ -0,0 +1,30 @@ +# NCBI MedCPT Cross-Encoder Reranker + +Deploy [ncbi/MedCPT-Cross-Encoder](https://huggingface.co/ncbi/MedCPT-Cross-Encoder) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [ncbi/MedCPT-Cross-Encoder](https://huggingface.co/ncbi/MedCPT-Cross-Encoder) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker/config.yaml b/embeddings/bei/ncbi-medcpt-cross-encoder-reranker/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker/config.yaml rename to embeddings/bei/ncbi-medcpt-cross-encoder-reranker/config.yaml index b283fb6ff..e29873927 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-ncbi-medcpt-cross-encoder-reranker/config.yaml +++ b/embeddings/bei/ncbi-medcpt-cross-encoder-reranker/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "ncbi/MedCPT-Cross-Encoder reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: query: What is Baseten? raw_scores: true diff --git a/embeddings/bei/ner-bert-base-ner-uncased/README.md b/embeddings/bei/ner-bert-base-ner-uncased/README.md new file mode 100644 index 000000000..daa8077a0 --- /dev/null +++ b/embeddings/bei/ner-bert-base-ner-uncased/README.md @@ -0,0 +1,30 @@ +# BERT Base NER Uncased Classification + +Deploy [baseten-admin/bert-base-ner-uncased](https://huggingface.co/baseten-admin/bert-base-ner-uncased) for classification using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [baseten-admin/bert-base-ner-uncased](https://huggingface.co/baseten-admin/bert-base-ner-uncased) | +| Task | Classification | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased/config.yaml b/embeddings/bei/ner-bert-base-ner-uncased/config.yaml similarity index 87% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased/config.yaml rename to embeddings/bei/ner-bert-base-ner-uncased/config.yaml index af374e453..c4daafbb1 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-ner-bert-base-ner-uncased/config.yaml +++ b/embeddings/bei/ner-bert-base-ner-uncased/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BERT Base NER Uncased named entity recognition model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Apple is looking at buying U.K. startup for $1 billion diff --git a/embeddings/bei/nomic-ai-nomic-embed-code-fp8/README.md b/embeddings/bei/nomic-ai-nomic-embed-code-fp8/README.md new file mode 100644 index 000000000..a153cc7bf --- /dev/null +++ b/embeddings/bei/nomic-ai-nomic-embed-code-fp8/README.md @@ -0,0 +1,31 @@ +# Nomic AI Nomic Embed Code + +Deploy [nomic-ai/nomic-embed-code](https://huggingface.co/nomic-ai/nomic-embed-code) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nomic-ai/nomic-embed-code](https://huggingface.co/nomic-ai/nomic-embed-code) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "nomic-ai/nomic-embed-code"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8/config.yaml b/embeddings/bei/nomic-ai-nomic-embed-code-fp8/config.yaml similarity index 85% rename from 11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8/config.yaml rename to embeddings/bei/nomic-ai-nomic-embed-code-fp8/config.yaml index 881343a19..7f967e657 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-nomic-ai-nomic-embed-code-fp8/config.yaml +++ b/embeddings/bei/nomic-ai-nomic-embed-code-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "nomic-ai/nomic-embed-code embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/nomic-ai-nomic-embed-text-v1.5/README.md b/embeddings/bei/nomic-ai-nomic-embed-text-v1.5/README.md new file mode 100644 index 000000000..4aced1333 --- /dev/null +++ b/embeddings/bei/nomic-ai-nomic-embed-text-v1.5/README.md @@ -0,0 +1,30 @@ +# Nomic AI Nomic Embed Text v1.5 + +Deploy [nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "nomic-ai/nomic-embed-text-v1.5"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5/config.yaml b/embeddings/bei/nomic-ai-nomic-embed-text-v1.5/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5/config.yaml rename to embeddings/bei/nomic-ai-nomic-embed-text-v1.5/config.yaml index de83f65ed..a1616f62d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v1.5/config.yaml +++ b/embeddings/bei/nomic-ai-nomic-embed-text-v1.5/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "nomic-ai/nomic-embed-text-v1.5 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/nomic-ai-nomic-embed-text-v2-moe/README.md b/embeddings/bei/nomic-ai-nomic-embed-text-v2-moe/README.md new file mode 100644 index 000000000..b836ddc4d --- /dev/null +++ b/embeddings/bei/nomic-ai-nomic-embed-text-v2-moe/README.md @@ -0,0 +1,30 @@ +# Nomic AI Nomic Embed Text v2 MoE + +Deploy [nomic-ai/nomic-embed-text-v2-moe](https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nomic-ai/nomic-embed-text-v2-moe](https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "nomic-ai/nomic-embed-text-v2-moe"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe/config.yaml b/embeddings/bei/nomic-ai-nomic-embed-text-v2-moe/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe/config.yaml rename to embeddings/bei/nomic-ai-nomic-embed-text-v2-moe/config.yaml index 481f63362..5c263ec3f 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nomic-ai-nomic-embed-text-v2-moe/config.yaml +++ b/embeddings/bei/nomic-ai-nomic-embed-text-v2-moe/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "nomic-ai/nomic-embed-text-v2-moe embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/nvidia-llama-embed-nemotron-8b/README.md b/embeddings/bei/nvidia-llama-embed-nemotron-8b/README.md new file mode 100644 index 000000000..dc93b9d02 --- /dev/null +++ b/embeddings/bei/nvidia-llama-embed-nemotron-8b/README.md @@ -0,0 +1,30 @@ +# NVIDIA Llama Embed Nemotron 8B + +Deploy [nvidia/llama-embed-nemotron-8b](https://huggingface.co/nvidia/llama-embed-nemotron-8b) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/llama-embed-nemotron-8b](https://huggingface.co/nvidia/llama-embed-nemotron-8b) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "nvidia/llama-embed-nemotron-8b"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b/config.yaml b/embeddings/bei/nvidia-llama-embed-nemotron-8b/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b/config.yaml rename to embeddings/bei/nvidia-llama-embed-nemotron-8b/config.yaml index e54df7d78..bcf7bb234 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-embed-nemotron-8b/config.yaml +++ b/embeddings/bei/nvidia-llama-embed-nemotron-8b/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "nvidia/llama-embed-nemotron-8b embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/nvidia-llama-nemotron-embed-1b-v2/README.md b/embeddings/bei/nvidia-llama-nemotron-embed-1b-v2/README.md new file mode 100644 index 000000000..88abdaa96 --- /dev/null +++ b/embeddings/bei/nvidia-llama-nemotron-embed-1b-v2/README.md @@ -0,0 +1,30 @@ +# NVIDIA Llama Nemotron Embed 1B v2 + +Deploy [nvidia/llama-nemotron-embed-1b-v2](https://huggingface.co/nvidia/llama-nemotron-embed-1b-v2) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/llama-nemotron-embed-1b-v2](https://huggingface.co/nvidia/llama-nemotron-embed-1b-v2) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "nvidia/llama-nemotron-embed-1b-v2"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2/config.yaml b/embeddings/bei/nvidia-llama-nemotron-embed-1b-v2/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2/config.yaml rename to embeddings/bei/nvidia-llama-nemotron-embed-1b-v2/config.yaml index 6741abc99..af49d4d68 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-nvidia-llama-nemotron-embed-1b-v2/config.yaml +++ b/embeddings/bei/nvidia-llama-nemotron-embed-1b-v2/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "nvidia/llama-nemotron-embed-1b-v2 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/papluca-xlm-roberta-base-language-detection-classification/README.md b/embeddings/bei/papluca-xlm-roberta-base-language-detection-classification/README.md new file mode 100644 index 000000000..574205f9d --- /dev/null +++ b/embeddings/bei/papluca-xlm-roberta-base-language-detection-classification/README.md @@ -0,0 +1,42 @@ +# Papluca XLM-RoBERTa Base Language Detection Classification + +Deploy [papluca/xlm-roberta-base-language-detection](https://huggingface.co/papluca/xlm-roberta-base-language-detection) for classification using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [papluca/xlm-roberta-base-language-detection](https://huggingface.co/papluca/xlm-roberta-base-language-detection) | +| Task | Classification | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "inputs": [ + [ + "Baseten is a fast inference provider" + ], + [ + "Classify this separately." + ] + ], + "raw_scores": true, + "truncate": true, + "truncation_direction": "Right" +}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification/config.yaml b/embeddings/bei/papluca-xlm-roberta-base-language-detection-classification/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification/config.yaml rename to embeddings/bei/papluca-xlm-roberta-base-language-detection-classification/config.yaml index 1a97785cb..5c454a3f0 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-papluca-xlm-roberta-base-language-detection-classification/config.yaml +++ b/embeddings/bei/papluca-xlm-roberta-base-language-detection-classification/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "papluca/xlm-roberta-base-language-detection classifier model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/qwen-qwen3-embedding-0.6b-fp8/README.md b/embeddings/bei/qwen-qwen3-embedding-0.6b-fp8/README.md new file mode 100644 index 000000000..e447f59e5 --- /dev/null +++ b/embeddings/bei/qwen-qwen3-embedding-0.6b-fp8/README.md @@ -0,0 +1,31 @@ +# Qwen3 Embedding 0.6B + +Deploy [michaelfeil/Qwen3-Embedding-0.6B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-0.6B-auto) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Embedding-0.6B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-0.6B-auto) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "michaelfeil/Qwen3-Embedding-0.6B-auto"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8/config.yaml b/embeddings/bei/qwen-qwen3-embedding-0.6b-fp8/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8/config.yaml rename to embeddings/bei/qwen-qwen3-embedding-0.6b-fp8/config.yaml index a80a34787..a3db4d55d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8/config.yaml +++ b/embeddings/bei/qwen-qwen3-embedding-0.6b-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Embedding-0.6B-auto embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/qwen-qwen3-embedding-4b-fp4/README.md b/embeddings/bei/qwen-qwen3-embedding-4b-fp4/README.md new file mode 100644 index 000000000..24da50130 --- /dev/null +++ b/embeddings/bei/qwen-qwen3-embedding-4b-fp4/README.md @@ -0,0 +1,31 @@ +# Qwen3 Embedding 4B FP4 + +Deploy [michaelfeil/Qwen3-Embedding-4B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-4B-auto) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Embedding-4B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-4B-auto) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | B200 | +| Quantization | FP4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "michaelfeil/Qwen3-Embedding-4B-auto"}' +``` + +## Configuration highlights + +- Quantization: **fp4** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4/config.yaml b/embeddings/bei/qwen-qwen3-embedding-4b-fp4/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4/config.yaml rename to embeddings/bei/qwen-qwen3-embedding-4b-fp4/config.yaml index 9ebddcf57..ee7823199 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp4/config.yaml +++ b/embeddings/bei/qwen-qwen3-embedding-4b-fp4/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Embedding-4B-auto embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/qwen-qwen3-embedding-4b-fp8/README.md b/embeddings/bei/qwen-qwen3-embedding-4b-fp8/README.md new file mode 100644 index 000000000..d810134b8 --- /dev/null +++ b/embeddings/bei/qwen-qwen3-embedding-4b-fp8/README.md @@ -0,0 +1,31 @@ +# Qwen3 Embedding 4B FP8 + +Deploy [michaelfeil/Qwen3-Embedding-4B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-4B-auto) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Embedding-4B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-4B-auto) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "michaelfeil/Qwen3-Embedding-4B-auto"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8/config.yaml b/embeddings/bei/qwen-qwen3-embedding-4b-fp8/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8/config.yaml rename to embeddings/bei/qwen-qwen3-embedding-4b-fp8/config.yaml index 21b3da96b..b0b0aa5e0 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8/config.yaml +++ b/embeddings/bei/qwen-qwen3-embedding-4b-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Embedding-4B-auto embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/qwen-qwen3-embedding-8b-fp8/README.md b/embeddings/bei/qwen-qwen3-embedding-8b-fp8/README.md new file mode 100644 index 000000000..8c58f9763 --- /dev/null +++ b/embeddings/bei/qwen-qwen3-embedding-8b-fp8/README.md @@ -0,0 +1,31 @@ +# Qwen3 Embedding 8B + +Deploy [michaelfeil/Qwen3-Embedding-8B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-8B-auto) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Embedding-8B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-8B-auto) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "michaelfeil/Qwen3-Embedding-8B-auto"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8/config.yaml b/embeddings/bei/qwen-qwen3-embedding-8b-fp8/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8/config.yaml rename to embeddings/bei/qwen-qwen3-embedding-8b-fp8/config.yaml index f7b30d76a..8abbbd778 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8/config.yaml +++ b/embeddings/bei/qwen-qwen3-embedding-8b-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Embedding-8B-auto embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/qwen-qwen3-reranker-0.6b-fp8/README.md b/embeddings/bei/qwen-qwen3-reranker-0.6b-fp8/README.md new file mode 100644 index 000000000..33f1e95ea --- /dev/null +++ b/embeddings/bei/qwen-qwen3-reranker-0.6b-fp8/README.md @@ -0,0 +1,31 @@ +# Qwen3 Reranker 0.6B + +Deploy [michaelfeil/Qwen3-Reranker-0.6B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-0.6B-seq) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Reranker-0.6B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-0.6B-seq) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8/config.yaml b/embeddings/bei/qwen-qwen3-reranker-0.6b-fp8/config.yaml similarity index 85% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8/config.yaml rename to embeddings/bei/qwen-qwen3-reranker-0.6b-fp8/config.yaml index de310dbeb..42d35697d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8/config.yaml +++ b/embeddings/bei/qwen-qwen3-reranker-0.6b-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Reranker-0.6B-seq reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/qwen-qwen3-reranker-4b-fp8/README.md b/embeddings/bei/qwen-qwen3-reranker-4b-fp8/README.md new file mode 100644 index 000000000..848395237 --- /dev/null +++ b/embeddings/bei/qwen-qwen3-reranker-4b-fp8/README.md @@ -0,0 +1,31 @@ +# Qwen3 Reranker 4B + +Deploy [michaelfeil/Qwen3-Reranker-4B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-4B-seq) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Reranker-4B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-4B-seq) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8/config.yaml b/embeddings/bei/qwen-qwen3-reranker-4b-fp8/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8/config.yaml rename to embeddings/bei/qwen-qwen3-reranker-4b-fp8/config.yaml index 0d85c1a24..e06fcb410 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8/config.yaml +++ b/embeddings/bei/qwen-qwen3-reranker-4b-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Reranker-4B-seq reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/qwen-qwen3-reranker-8b-fp4/README.md b/embeddings/bei/qwen-qwen3-reranker-8b-fp4/README.md new file mode 100644 index 000000000..32cbc9dc0 --- /dev/null +++ b/embeddings/bei/qwen-qwen3-reranker-8b-fp4/README.md @@ -0,0 +1,31 @@ +# Qwen3 Reranker 8B FP4 + +Deploy [michaelfeil/Qwen3-Reranker-8B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-8B-seq) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Reranker-8B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-8B-seq) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | B200 | +| Quantization | FP4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Quantization: **fp4** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4/config.yaml b/embeddings/bei/qwen-qwen3-reranker-8b-fp4/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4/config.yaml rename to embeddings/bei/qwen-qwen3-reranker-8b-fp4/config.yaml index a15f890df..f955c1593 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp4/config.yaml +++ b/embeddings/bei/qwen-qwen3-reranker-8b-fp4/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Reranker-8B-seq reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/qwen-qwen3-reranker-8b-fp8/README.md b/embeddings/bei/qwen-qwen3-reranker-8b-fp8/README.md new file mode 100644 index 000000000..60d6aebb5 --- /dev/null +++ b/embeddings/bei/qwen-qwen3-reranker-8b-fp8/README.md @@ -0,0 +1,31 @@ +# Qwen3 Reranker 8B FP8 + +Deploy [michaelfeil/Qwen3-Reranker-8B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-8B-seq) as a reranker using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/Qwen3-Reranker-8B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-8B-seq) | +| Task | Reranking | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8/config.yaml b/embeddings/bei/qwen-qwen3-reranker-8b-fp8/config.yaml similarity index 86% rename from 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8/config.yaml rename to embeddings/bei/qwen-qwen3-reranker-8b-fp8/config.yaml index cb254db4d..c807cc5fb 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8/config.yaml +++ b/embeddings/bei/qwen-qwen3-reranker-8b-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "michaelfeil/Qwen3-Reranker-8B-seq reranker model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/redis-langcache-embed-v2/README.md b/embeddings/bei/redis-langcache-embed-v2/README.md new file mode 100644 index 000000000..2bed91e4a --- /dev/null +++ b/embeddings/bei/redis-langcache-embed-v2/README.md @@ -0,0 +1,30 @@ +# Redis LangCache Embed v2 + +Deploy [redis/langcache-embed-v2](https://huggingface.co/redis/langcache-embed-v2) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [redis/langcache-embed-v2](https://huggingface.co/redis/langcache-embed-v2) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "redis/langcache-embed-v2"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2/config.yaml b/embeddings/bei/redis-langcache-embed-v2/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2/config.yaml rename to embeddings/bei/redis-langcache-embed-v2/config.yaml index 5055c160f..dcefdf264 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-redis-langcache-embed-v2/config.yaml +++ b/embeddings/bei/redis-langcache-embed-v2/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "redis/langcache-embed-v2 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/salesforce-sfr-embedding-mistral-fp8/README.md b/embeddings/bei/salesforce-sfr-embedding-mistral-fp8/README.md new file mode 100644 index 000000000..5c211bdd9 --- /dev/null +++ b/embeddings/bei/salesforce-sfr-embedding-mistral-fp8/README.md @@ -0,0 +1,31 @@ +# Salesforce SFR Embedding Mistral + +Deploy [Salesforce/SFR-Embedding-Mistral](https://huggingface.co/Salesforce/SFR-Embedding-Mistral) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Salesforce/SFR-Embedding-Mistral](https://huggingface.co/Salesforce/SFR-Embedding-Mistral) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Salesforce/SFR-Embedding-Mistral"}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8/config.yaml b/embeddings/bei/salesforce-sfr-embedding-mistral-fp8/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8/config.yaml rename to embeddings/bei/salesforce-sfr-embedding-mistral-fp8/config.yaml index ccd3c42d9..74c92696f 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-salesforce-sfr-embedding-mistral-fp8/config.yaml +++ b/embeddings/bei/salesforce-sfr-embedding-mistral-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Salesforce/SFR-Embedding-Mistral embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/samlowe-roberta-base-go_emotions-classification/README.md b/embeddings/bei/samlowe-roberta-base-go_emotions-classification/README.md new file mode 100644 index 000000000..8002a837c --- /dev/null +++ b/embeddings/bei/samlowe-roberta-base-go_emotions-classification/README.md @@ -0,0 +1,42 @@ +# SamLowe RoBERTa Base GoEmotions Classification + +Deploy [SamLowe/roberta-base-go_emotions](https://huggingface.co/SamLowe/roberta-base-go_emotions) for classification using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [SamLowe/roberta-base-go_emotions](https://huggingface.co/SamLowe/roberta-base-go_emotions) | +| Task | Classification | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "inputs": [ + [ + "Baseten is a fast inference provider" + ], + [ + "Classify this separately." + ] + ], + "raw_scores": true, + "truncate": true, + "truncation_direction": "Right" +}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification/config.yaml b/embeddings/bei/samlowe-roberta-base-go_emotions-classification/config.yaml similarity index 85% rename from 11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification/config.yaml rename to embeddings/bei/samlowe-roberta-base-go_emotions-classification/config.yaml index 966d5a3c5..c0e5048af 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-samlowe-roberta-base-go_emotions-classification/config.yaml +++ b/embeddings/bei/samlowe-roberta-base-go_emotions-classification/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "SamLowe/roberta-base-go_emotions classifier model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding/README.md b/embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding/README.md new file mode 100644 index 000000000..ed45b7dcd --- /dev/null +++ b/embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding/README.md @@ -0,0 +1,30 @@ +# Sentence Transformers All-MiniLM-L6-v2 Embedding + +Deploy [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "sentence-transformers/all-MiniLM-L6-v2"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding/config.yaml b/embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding/config.yaml similarity index 83% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding/config.yaml rename to embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding/config.yaml index bd676606e..2e60c4570 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-sentence-transformers-all-minilm-l6-v2-embedding/config.yaml +++ b/embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "sentence-transformers/all-MiniLM-L6-v2 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md b/embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md new file mode 100644 index 000000000..d13af8b3b --- /dev/null +++ b/embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/README.md @@ -0,0 +1,43 @@ +# Skywork Reward Llama 3.1 8B v0.2 + +Deploy [Skywork/Skywork-Reward-Llama-3.1-8B-v0.2](https://huggingface.co/Skywork/Skywork-Reward-Llama-3.1-8B-v0.2) for classification using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Skywork/Skywork-Reward-Llama-3.1-8B-v0.2](https://huggingface.co/Skywork/Skywork-Reward-Llama-3.1-8B-v0.2) | +| Task | Classification | +| Engine | BEI (TensorRT) | +| GPU | H100_40GB | +| Quantization | FP8 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "inputs": [ + [ + "Baseten is a fast inference provider" + ], + [ + "Classify this separately." + ] + ], + "raw_scores": true, + "truncate": true, + "truncation_direction": "Right" +}' +``` + +## Configuration highlights + +- Quantization: **fp8** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/config.yaml b/embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/config.yaml similarity index 85% rename from 11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/config.yaml rename to embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/config.yaml index 43afb3725..22829dae1 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/config.yaml +++ b/embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 classifier model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Baseten is a fast inference provider diff --git a/embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0/README.md b/embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0/README.md new file mode 100644 index 000000000..ecd0abe55 --- /dev/null +++ b/embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0/README.md @@ -0,0 +1,30 @@ +# Snowflake Arctic Embed L v2.0 + +Deploy [Snowflake/snowflake-arctic-embed-l-v2.0](https://huggingface.co/Snowflake/snowflake-arctic-embed-l-v2.0) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Snowflake/snowflake-arctic-embed-l-v2.0](https://huggingface.co/Snowflake/snowflake-arctic-embed-l-v2.0) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Snowflake/snowflake-arctic-embed-l-v2.0"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0/config.yaml b/embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0/config.yaml similarity index 82% rename from 11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0/config.yaml rename to embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0/config.yaml index 20d35c634..394d3c0b0 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-snowflake-snowflake-arctic-embed-l-v2.0/config.yaml +++ b/embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Snowflake/snowflake-arctic-embed-l-v2.0 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/tanaos-tanaos-ner-v1/README.md b/embeddings/bei/tanaos-tanaos-ner-v1/README.md new file mode 100644 index 000000000..8219220db --- /dev/null +++ b/embeddings/bei/tanaos-tanaos-ner-v1/README.md @@ -0,0 +1,30 @@ +# Tanaos NER v1 + +Deploy [tanaos/tanaos-NER-v1](https://huggingface.co/tanaos/tanaos-NER-v1) for classification using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [tanaos/tanaos-NER-v1](https://huggingface.co/tanaos/tanaos-NER-v1) | +| Task | Classification | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1/config.yaml b/embeddings/bei/tanaos-tanaos-ner-v1/config.yaml similarity index 87% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1/config.yaml rename to embeddings/bei/tanaos-tanaos-ner-v1/config.yaml index 1354122c8..025cbdecf 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-tanaos-tanaos-ner-v1/config.yaml +++ b/embeddings/bei/tanaos-tanaos-ner-v1/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Tanaos NER v1 named entity recognition model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: inputs: - - Apple is looking at buying U.K. startup for $1 billion diff --git a/embeddings/bei/taylorai-bge-micro-v2/README.md b/embeddings/bei/taylorai-bge-micro-v2/README.md new file mode 100644 index 000000000..07c3c3eca --- /dev/null +++ b/embeddings/bei/taylorai-bge-micro-v2/README.md @@ -0,0 +1,30 @@ +# TaylorAI BGE Micro v2 + +Deploy [TaylorAI/bge-micro-v2](https://huggingface.co/TaylorAI/bge-micro-v2) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [TaylorAI/bge-micro-v2](https://huggingface.co/TaylorAI/bge-micro-v2) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "TaylorAI/bge-micro-v2"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2/config.yaml b/embeddings/bei/taylorai-bge-micro-v2/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2/config.yaml rename to embeddings/bei/taylorai-bge-micro-v2/config.yaml index e03d912f4..6f0f4848d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-taylorai-bge-micro-v2/config.yaml +++ b/embeddings/bei/taylorai-bge-micro-v2/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "TaylorAI/bge-micro-v2 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/voyageai-voyage-4-nano/README.md b/embeddings/bei/voyageai-voyage-4-nano/README.md new file mode 100644 index 000000000..9f95a2482 --- /dev/null +++ b/embeddings/bei/voyageai-voyage-4-nano/README.md @@ -0,0 +1,30 @@ +# VoyageAI Voyage 4 Nano + +Deploy [voyageai/voyage-4-nano](https://huggingface.co/voyageai/voyage-4-nano) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [voyageai/voyage-4-nano](https://huggingface.co/voyageai/voyage-4-nano) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "voyageai/voyage-4-nano"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano/config.yaml b/embeddings/bei/voyageai-voyage-4-nano/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano/config.yaml rename to embeddings/bei/voyageai-voyage-4-nano/config.yaml index 6d709999a..944d2befc 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-Bert-voyageai-voyage-4-nano/config.yaml +++ b/embeddings/bei/voyageai-voyage-4-nano/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "voyageai/voyage-4-nano embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/bei/whereisai-uae-large-v1-embedding/README.md b/embeddings/bei/whereisai-uae-large-v1-embedding/README.md new file mode 100644 index 000000000..790e4cb91 --- /dev/null +++ b/embeddings/bei/whereisai-uae-large-v1-embedding/README.md @@ -0,0 +1,30 @@ +# WhereIsAI UAE Large V1 Embedding + +Deploy [WhereIsAI/UAE-Large-V1](https://huggingface.co/WhereIsAI/UAE-Large-V1) for generating text embeddings using a BEI (TensorRT) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [WhereIsAI/UAE-Large-V1](https://huggingface.co/WhereIsAI/UAE-Large-V1) | +| Task | Embeddings | +| Engine | BEI (TensorRT) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "WhereIsAI/UAE-Large-V1"}' +``` + +## Configuration highlights + +- Engine: **BEI (TensorRT)** diff --git a/11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding/config.yaml b/embeddings/bei/whereisai-uae-large-v1-embedding/config.yaml similarity index 84% rename from 11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding/config.yaml rename to embeddings/bei/whereisai-uae-large-v1-embedding/config.yaml index 5e490a51b..2ae321c7f 100644 --- a/11-embeddings-reranker-classification-tensorrt/BEI-whereisai-uae-large-v1-embedding/config.yaml +++ b/embeddings/bei/whereisai-uae-large-v1-embedding/config.yaml @@ -1,5 +1,8 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "WhereIsAI/UAE-Large-V1 embedding model" model_metadata: + tags: + - force-legacy-api-non-openai-compatible example_model_input: encoding_format: float input: text string diff --git a/embeddings/clip/README.md b/embeddings/clip/README.md new file mode 100644 index 000000000..ce5715828 --- /dev/null +++ b/embeddings/clip/README.md @@ -0,0 +1,31 @@ +# clip-example + +Deploy clip-example for generating text embeddings using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Embeddings | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://images.pexels.com/photos/1170986/pexels-photo-1170986.jpeg?auto=compress&cs=tinysrgb&w=1600" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/clip/config.yaml b/embeddings/clip/config.yaml similarity index 76% rename from clip/config.yaml rename to embeddings/clip/config.yaml index 873618ceb..450b28a8f 100644 --- a/clip/config.yaml +++ b/embeddings/clip/config.yaml @@ -1,14 +1,16 @@ +description: "CLIP for image-text embedding" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "openai/clip-vit-large-patch14" example_model_input: url: https://images.pexels.com/photos/1170986/pexels-photo-1170986.jpeg?auto=compress&cs=tinysrgb&w=1600 model_name: clip-example python_version: py311 requirements: - transformers==4.47.1 -- pillow -- torch +- pillow==10.1.0 +- torch==2.1.0 resources: accelerator: A10G cpu: '3' diff --git a/flux.1-dev-trt-b200/model/__init__.py b/embeddings/clip/model/__init__.py similarity index 100% rename from flux.1-dev-trt-b200/model/__init__.py rename to embeddings/clip/model/__init__.py diff --git a/clip/model/model.py b/embeddings/clip/model/model.py similarity index 100% rename from clip/model/model.py rename to embeddings/clip/model/model.py diff --git a/embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding/README.md b/embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding/README.md new file mode 100644 index 000000000..821df1ca9 --- /dev/null +++ b/embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding/README.md @@ -0,0 +1,32 @@ +# Alibaba-NLP GTE ModernBERT Base Embedding + +Deploy [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Alibaba-NLP/gte-modernbert-base"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding/config.yaml b/embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding/config.yaml rename to embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding/config.yaml index bf026acfe..5833f5282 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-modernbert-base-embedding/config.yaml +++ b/embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-modernbert-base embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md b/embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md new file mode 100644 index 000000000..a3d32f5e9 --- /dev/null +++ b/embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/README.md @@ -0,0 +1,32 @@ +# Alibaba-NLP GTE Qwen2 1.5B Instruct Embedding + +Deploy [Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml b/embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml rename to embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml index 4b5904a87..b7afa13fa 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml +++ b/embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-Qwen2-1.5B-instruct embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md b/embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md new file mode 100644 index 000000000..5eb543a6e --- /dev/null +++ b/embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/README.md @@ -0,0 +1,32 @@ +# Alibaba-NLP GTE Qwen2 7B Instruct Embedding + +Deploy [Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "Alibaba-NLP/gte-Qwen2-7B-instruct"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:hopper-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml b/embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml rename to embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml index 98e3c0c98..bcd3dbfe6 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml +++ b/embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-Qwen2-7B-instruct embedding model" base_image: image: baseten/text-embeddings-inference-mirror:hopper-1.8.3 docker_server: diff --git a/embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base/README.md b/embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base/README.md new file mode 100644 index 000000000..46552d612 --- /dev/null +++ b/embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base/README.md @@ -0,0 +1,32 @@ +# Alibaba-NLP GTE Reranker ModernBERT Base + +Deploy [Alibaba-NLP/gte-reranker-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base) as a reranker using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Alibaba-NLP/gte-reranker-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base) | +| Task | Reranking | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base/config.yaml b/embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base/config.yaml rename to embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base/config.yaml index daa6fa816..6e5d26047 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-alibaba-nlp-gte-reranker-modernbert-base/config.yaml +++ b/embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Alibaba-NLP/gte-reranker-modernbert-base reranker model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/baai-bge-reranker-large/README.md b/embeddings/tei/baai-bge-reranker-large/README.md new file mode 100644 index 000000000..87e674306 --- /dev/null +++ b/embeddings/tei/baai-bge-reranker-large/README.md @@ -0,0 +1,32 @@ +# BAAI BGE Reranker Large + +Deploy [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) as a reranker using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) | +| Task | Reranking | +| Engine | TEI (HuggingFace) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/rerank \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "What is deep learning?", "texts": ["Deep learning is a subset of machine learning.", "The weather is nice today."], "raw_scores": true}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:hopper-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large/config.yaml b/embeddings/tei/baai-bge-reranker-large/config.yaml similarity index 95% rename from 11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large/config.yaml rename to embeddings/tei/baai-bge-reranker-large/config.yaml index 188e5838a..cabfa4eaf 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-baai-bge-reranker-large/config.yaml +++ b/embeddings/tei/baai-bge-reranker-large/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "BAAI/bge-reranker-large reranker model" base_image: image: baseten/text-embeddings-inference-mirror:hopper-1.8.3 docker_server: diff --git a/embeddings/tei/google-embeddinggemma-300m/README.md b/embeddings/tei/google-embeddinggemma-300m/README.md new file mode 100644 index 000000000..a919671af --- /dev/null +++ b/embeddings/tei/google-embeddinggemma-300m/README.md @@ -0,0 +1,32 @@ +# Google EmbeddingGemma 300M Embedding + +Deploy [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "google/embeddinggemma-300m"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m/config.yaml b/embeddings/tei/google-embeddinggemma-300m/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m/config.yaml rename to embeddings/tei/google-embeddinggemma-300m/config.yaml index 9250697ef..14fd09943 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-google-embeddinggemma-300m/config.yaml +++ b/embeddings/tei/google-embeddinggemma-300m/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "google/embeddinggemma-300m embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/intfloat-multilingual-e5-large-instruct/README.md b/embeddings/tei/intfloat-multilingual-e5-large-instruct/README.md new file mode 100644 index 000000000..9611a669f --- /dev/null +++ b/embeddings/tei/intfloat-multilingual-e5-large-instruct/README.md @@ -0,0 +1,32 @@ +# Intfloat Multilingual E5 Large Instruct Embedding + +Deploy [intfloat/multilingual-e5-large-instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [intfloat/multilingual-e5-large-instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "intfloat/multilingual-e5-large-instruct"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct/config.yaml b/embeddings/tei/intfloat-multilingual-e5-large-instruct/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct/config.yaml rename to embeddings/tei/intfloat-multilingual-e5-large-instruct/config.yaml index 98746c8d8..7787cd1cd 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-intfloat-multilingual-e5-large-instruct/config.yaml +++ b/embeddings/tei/intfloat-multilingual-e5-large-instruct/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "intfloat/multilingual-e5-large-instruct embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/jina-ai-jina-embeddings-v2-base-en/README.md b/embeddings/tei/jina-ai-jina-embeddings-v2-base-en/README.md new file mode 100644 index 000000000..3e7078bd1 --- /dev/null +++ b/embeddings/tei/jina-ai-jina-embeddings-v2-base-en/README.md @@ -0,0 +1,32 @@ +# Jina AI Jina Embeddings v2 Base EN + +Deploy [jinaai/jina-embeddings-v2-base-en](https://huggingface.co/jinaai/jina-embeddings-v2-base-en) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [jinaai/jina-embeddings-v2-base-en](https://huggingface.co/jinaai/jina-embeddings-v2-base-en) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "jinaai/jina-embeddings-v2-base-en"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en/config.yaml b/embeddings/tei/jina-ai-jina-embeddings-v2-base-en/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en/config.yaml rename to embeddings/tei/jina-ai-jina-embeddings-v2-base-en/config.yaml index e68e8c953..f69abfc71 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-jina-ai-jina-embeddings-v2-base-en/config.yaml +++ b/embeddings/tei/jina-ai-jina-embeddings-v2-base-en/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "jinaai/jina-embeddings-v2-base-en embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/jinaai-jina-embeddings-v2-base-code/README.md b/embeddings/tei/jinaai-jina-embeddings-v2-base-code/README.md new file mode 100644 index 000000000..8a85a0556 --- /dev/null +++ b/embeddings/tei/jinaai-jina-embeddings-v2-base-code/README.md @@ -0,0 +1,32 @@ +# Jina AI Jina Embeddings v2 Base Code + +Deploy [jinaai/jina-embeddings-v2-base-code](https://huggingface.co/jinaai/jina-embeddings-v2-base-code) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [jinaai/jina-embeddings-v2-base-code](https://huggingface.co/jinaai/jina-embeddings-v2-base-code) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "jinaai/jina-embeddings-v2-base-code"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code/config.yaml b/embeddings/tei/jinaai-jina-embeddings-v2-base-code/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code/config.yaml rename to embeddings/tei/jinaai-jina-embeddings-v2-base-code/config.yaml index ac49be155..acd5f4826 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-jinaai-jina-embeddings-v2-base-code/config.yaml +++ b/embeddings/tei/jinaai-jina-embeddings-v2-base-code/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "jinaai/jina-embeddings-v2-base-code embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md b/embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md new file mode 100644 index 000000000..b2573e0f0 --- /dev/null +++ b/embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding/README.md @@ -0,0 +1,32 @@ +# Mixedbread AI MxBAI Embed Large v1 Embedding + +Deploy [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mixedbread-ai/mxbai-embed-large-v1](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "mixedbread-ai/mxbai-embed-large-v1"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml b/embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml rename to embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml index 57b066b85..7c1cf267f 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml +++ b/embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "mixedbread-ai/mxbai-embed-large-v1 embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/nomic-ai-nomic-embed-text-v1.5/README.md b/embeddings/tei/nomic-ai-nomic-embed-text-v1.5/README.md new file mode 100644 index 000000000..cd90b155e --- /dev/null +++ b/embeddings/tei/nomic-ai-nomic-embed-text-v1.5/README.md @@ -0,0 +1,32 @@ +# Nomic AI Nomic Embed Text v1.5 + +Deploy [nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nomic-ai/nomic-embed-text-v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "nomic-ai/nomic-embed-text-v1.5"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:86-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5/config.yaml b/embeddings/tei/nomic-ai-nomic-embed-text-v1.5/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5/config.yaml rename to embeddings/tei/nomic-ai-nomic-embed-text-v1.5/config.yaml index 60998c346..4041a2b8e 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v1.5/config.yaml +++ b/embeddings/tei/nomic-ai-nomic-embed-text-v1.5/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "nomic-ai/nomic-embed-text-v1.5 embedding model" base_image: image: baseten/text-embeddings-inference-mirror:86-1.8.3 docker_server: diff --git a/embeddings/tei/nomic-ai-nomic-embed-text-v2-moe/README.md b/embeddings/tei/nomic-ai-nomic-embed-text-v2-moe/README.md new file mode 100644 index 000000000..2567e1724 --- /dev/null +++ b/embeddings/tei/nomic-ai-nomic-embed-text-v2-moe/README.md @@ -0,0 +1,32 @@ +# Nomic AI Nomic Embed Text v2 MoE + +Deploy [nomic-ai/nomic-embed-text-v2-moe](https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nomic-ai/nomic-embed-text-v2-moe](https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "nomic-ai/nomic-embed-text-v2-moe"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe/config.yaml b/embeddings/tei/nomic-ai-nomic-embed-text-v2-moe/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe/config.yaml rename to embeddings/tei/nomic-ai-nomic-embed-text-v2-moe/config.yaml index b035c847d..715bc3f8a 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-nomic-ai-nomic-embed-text-v2-moe/config.yaml +++ b/embeddings/tei/nomic-ai-nomic-embed-text-v2-moe/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "nomic-ai/nomic-embed-text-v2-moe embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/redis-langcache-embed-v2/README.md b/embeddings/tei/redis-langcache-embed-v2/README.md new file mode 100644 index 000000000..550801a53 --- /dev/null +++ b/embeddings/tei/redis-langcache-embed-v2/README.md @@ -0,0 +1,32 @@ +# Redis LangCache Embed v2 + +Deploy [redis/langcache-embed-v2](https://huggingface.co/redis/langcache-embed-v2) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [redis/langcache-embed-v2](https://huggingface.co/redis/langcache-embed-v2) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "redis/langcache-embed-v2"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2/config.yaml b/embeddings/tei/redis-langcache-embed-v2/config.yaml similarity index 95% rename from 11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2/config.yaml rename to embeddings/tei/redis-langcache-embed-v2/config.yaml index 3834cc1d7..7ead7013c 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-redis-langcache-embed-v2/config.yaml +++ b/embeddings/tei/redis-langcache-embed-v2/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "redis/langcache-embed-v2 embedding model" base_image: image: baseten/text-embeddings-inference-mirror:89-1.8.3 docker_server: diff --git a/embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding/README.md b/embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding/README.md new file mode 100644 index 000000000..06d53d936 --- /dev/null +++ b/embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding/README.md @@ -0,0 +1,32 @@ +# Sentence Transformers All-MiniLM-L6-v2 Embedding + +Deploy [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | T4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "sentence-transformers/all-MiniLM-L6-v2"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:turing-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding/config.yaml b/embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding/config.yaml rename to embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding/config.yaml index 050819f1c..1533f40a3 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-sentence-transformers-all-minilm-l6-v2-embedding/config.yaml +++ b/embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "sentence-transformers/all-MiniLM-L6-v2 embedding model" base_image: image: baseten/text-embeddings-inference-mirror:turing-1.8.3 docker_server: diff --git a/embeddings/tei/taylorai-bge-micro-v2/README.md b/embeddings/tei/taylorai-bge-micro-v2/README.md new file mode 100644 index 000000000..fd2071a80 --- /dev/null +++ b/embeddings/tei/taylorai-bge-micro-v2/README.md @@ -0,0 +1,32 @@ +# TaylorAI BGE Micro v2 + +Deploy [TaylorAI/bge-micro-v2](https://huggingface.co/TaylorAI/bge-micro-v2) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [TaylorAI/bge-micro-v2](https://huggingface.co/TaylorAI/bge-micro-v2) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "TaylorAI/bge-micro-v2"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:86-1.8.3` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2/config.yaml b/embeddings/tei/taylorai-bge-micro-v2/config.yaml similarity index 95% rename from 11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2/config.yaml rename to embeddings/tei/taylorai-bge-micro-v2/config.yaml index 54fb71b40..267755b81 100644 --- a/11-embeddings-reranker-classification-tensorrt/TEI-taylorai-bge-micro-v2/config.yaml +++ b/embeddings/tei/taylorai-bge-micro-v2/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "TaylorAI/bge-micro-v2 embedding model" base_image: image: baseten/text-embeddings-inference-mirror:86-1.8.3 docker_server: diff --git a/embeddings/text-embeddings-inference/README.md b/embeddings/text-embeddings-inference/README.md new file mode 100644 index 000000000..8d39c0145 --- /dev/null +++ b/embeddings/text-embeddings-inference/README.md @@ -0,0 +1,33 @@ +# text-embeddings-inference trussless + +Deploy [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5) for generating text embeddings using a TEI (HuggingFace) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5) | +| Task | Embeddings | +| Engine | TEI (HuggingFace) | +| GPU | L4 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"input": "What is deep learning?", "model": "BAAI/bge-base-en-v1.5"}' +``` + +## Configuration highlights + +- Base image: `baseten/text-embeddings-inference-mirror:89-1.6` +- Predict concurrency: **40** +- Server port: **7997** diff --git a/text-embeddings-inference/config.yaml b/embeddings/text-embeddings-inference/config.yaml similarity index 89% rename from text-embeddings-inference/config.yaml rename to embeddings/text-embeddings-inference/config.yaml index 82332a3ce..048a9f3cc 100644 --- a/text-embeddings-inference/config.yaml +++ b/embeddings/text-embeddings-inference/config.yaml @@ -1,3 +1,4 @@ +description: "BGE Base EN v1.5 via Text Embeddings Inference server" base_image: # select an image: L4 # CPU baseten/text-embeddings-inference-mirror:cpu-1.6 @@ -9,6 +10,7 @@ base_image: image: baseten/text-embeddings-inference-mirror:89-1.6 model_metadata: repo_id: BAAI/bge-base-en-v1.5 + example_model_input: {"input": "What is deep learning?", "model": "BAAI/bge-base-en-v1.5"} docker_server: start_command: sh -c "text-embeddings-router --port 7997 --model-id /data/local-model --max-client-batch-size 32 --max-concurrent-requests 40 --max-batch-tokens 32768" readiness_endpoint: /health @@ -26,5 +28,4 @@ build_commands: # optional step to download the weights of the model into the im runtime: predict_concurrency : 40 environment_variables: - VLLM_LOGGING_LEVEL: WARNING hf_access_token: null diff --git a/falcon/falcon3-10B-trt-llm-spec-dec/README.md b/falcon/falcon3-10B-trt-llm-spec-dec/README.md deleted file mode 100644 index 15b89047e..000000000 --- a/falcon/falcon3-10B-trt-llm-spec-dec/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Falcon 3 10B Instruct using TensorRT-LLM with Speculative Decoding - -This directory is [Truss](https://truss.baseten.co/) template for deploying model Llama 3.1 8B Instruct using our TensorRT-LLM (TRT-LLM) [engine builder](https://docs.baseten.co/performance/engine-builder-overview). This configuration is optimized for high-throughput scenarios. - -## Use case - -This deployment is tailored for applications that require processing large volumes of data with moderate to low latency, such as: -* Content moderation for social media platforms -* Bulk article summarization -* Large-scale Retrieval-Augmented Generation (RAG) systems - -## Configuration - -The template uses the following key configuration parameters: - -| Property | Value | Description | -| ------------------- | -------- | ------------------------------------------------------------------------------ | -| GPU | 1xH100 | Single NVIDIA H100 GPU | -| `max_batch_size` | 32 | Allows processing up to 32 requests simultaneously | -| `quantization_type` | `fp8_kv` | FP8 quantization, balancing performance and accuracy. See this [blog](https://www.baseten.co/blog/33-faster-llm-inference-with-fp8-quantization/). | -| `max_input_len` | 4096 | Maximum number of input tokens that the model will accept | -| `max_output_len` | 1024 | Maximum number of output tokens the model can generate | - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd falcon/falcon3-10B-trt-llm-spec-dec -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `falcon/falcon3-10B-trt-llm-spec-dec` as your working directory, you can deploy the model with: - -```sh -truss push --trusted --publish -``` - -Paste your Baseten API key if prompted. Also ensure the `hf_access_token` secret is properly setup in your Baseten Account to access this model. - -**Note**: TensorRT-LLM with engine builder will only work under a Baseten production deployment - -For more information, refer to the [Truss documentation](https://docs.baseten.co/performance/engine-builder-overview). diff --git a/falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md b/falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md deleted file mode 100644 index 9a832a539..000000000 --- a/falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Falcon 3.1 3B Instruct using TensorRT-LLM with High Throughput - -This directory is [Truss](https://truss.baseten.co/) template for deploying model Falcon 3.1 3B Instruct using our TensorRT-LLM (TRT-LLM) [engine builder](https://docs.baseten.co/performance/engine-builder-overview). This configuration is optimized for high-throughput scenarios. - -## Use case - -This deployment is tailored for applications that require processing large volumes of data with moderate to low latency, such as: -* Content moderation for social media platforms -* Bulk article summarization -* Large-scale Retrieval-Augmented Generation (RAG) systems - -## Configuration - -The template uses the following key configuration parameters: - -| Property | Value | Description | -| ------------------- | -------- | ------------------------------------------------------------------------------ | -| GPU | 1xH100 | Single NVIDIA H100 GPU | -| `max_batch_size` | 32 | Allows processing up to 32 requests simultaneously | -| `quantization_type` | `fp8_kv` | FP8 quantization, balancing performance and accuracy. See this [blog](https://www.baseten.co/blog/33-faster-llm-inference-with-fp8-quantization/). | -| `max_input_len` | 4096 | Maximum number of input tokens that the model will accept | -| `max_output_len` | 1024 | Maximum number of output tokens the model can generate | - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd falcon/falcon3-3B-trt-llm-engine-high-throughput -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `falcon/falcon3-3B-trt-llm-engine-high-throughput` as your working directory, you can deploy the model with: - -```sh -truss push --trusted --publish -``` - -Paste your Baseten API key if prompted. Also ensure the `hf_access_token` secret is properly setup in your Baseten Account to access this model. - -**Note**: TensorRT-LLM with engine builder will only work under a Baseten production deployment - -For more information, refer to the [Truss documentation](https://docs.baseten.co/performance/engine-builder-overview). diff --git a/flux.1-dev-trt-b200/README.md b/flux.1-dev-trt-b200/README.md deleted file mode 100644 index 80ff0cfbe..000000000 --- a/flux.1-dev-trt-b200/README.md +++ /dev/null @@ -1,280 +0,0 @@ -# Flux 1.0 Dev - TensorRT - -This model provides high-quality text-to-image generation using [Flux.1-dev model](https://huggingface.co/black-forest-labs/FLUX.1-dev) optimized with TensorRT for the B200 GPU. - -## Model Information - -- **Model**: Flux 1.0 Dev (black-forest-labs/FLUX.1-dev) -- **Optimization**: TensorRT 8.6.1 -- **Hardware**: NVIDIA B200 GPU -- **Framework**: PyTorch with TensorRT acceleration - -## Features - -- High-quality image generation from text prompts -- TensorRT optimization for fast inference -- Support for custom image dimensions (must be multiples of 8) -- Configurable denoising steps and guidance scale -- CUDA graph optimization support - -## Usage - -### Basic Usage - -```bash -curl -X POST https://app.baseten.co/models/{MODEL_ID}/predict \ - -H "Authorization: Api-Key API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "A beautiful landscape with mountains and a lake, photorealistic, high quality", - "negative_prompt": "blurry, low quality, distorted", - "height": 1024, - "width": 1024, - "num_inference_steps": 30, - "guidance_scale": 3.5, - "seed": 42, - "batch_size": 1, - "batch_count": 1 - }' | python show.py -``` - -### Batch Processing - -The model supports efficient batch processing for generating multiple images in a single request. You can generate up to 4 images simultaneously with either the same prompt or different prompts for each image. - -#### Same Prompt for All Images - -```bash -curl -X POST https://app.baseten.co/models/{MODEL_ID}/predict \ - -H "Authorization: Api-Key API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "A beautiful landscape with mountains and a lake, photorealistic, high quality", - "negative_prompt": "blurry, low quality, distorted", - "height": 1024, - "width": 1024, - "num_inference_steps": 30, - "guidance_scale": 3.5, - "seed": 42, - "batch_size": 4, - "batch_count": 1 - }' | python show_batch.py -``` - -#### Different Prompts for Each Image - -```bash -curl -X POST https://app.baseten.co/models/{MODEL_ID}/predict \ - -H "Authorization: Api-Key API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": [ - "A beautiful landscape with mountains and a lake, photorealistic, high quality", - "A futuristic city skyline at sunset, neon lights, cyberpunk style, high quality", - "A cute cat sitting in a garden, soft lighting, detailed, high quality", - "Abstract geometric patterns in vibrant colors, modern art style, high quality" - ], - "negative_prompt": [ - "blurry, low quality, distorted", - "blurry, low quality, distorted", - "blurry, low quality, distorted", - "blurry, low quality, distorted" - ], - "height": 1024, - "width": 1024, - "num_inference_steps": 30, - "guidance_scale": 3.5, - "seed": 42, - "batch_size": 4, - "batch_count": 1 - }' | python show_batch.py -``` - -#### Batch Processing Benefits - -- **Parallel Processing**: All images in a batch are generated simultaneously, not sequentially -- **Better GPU Utilization**: More efficient use of GPU resources compared to separate requests -- **Faster Total Time**: Generating 4 images in a batch is significantly faster than 4 separate API calls -- **Consistent Parameters**: All images in a batch use the same dimensions, inference steps, and guidance scale - -#### Batch Processing Limitations - -- **Maximum Batch Size**: Limited to 4 images per batch (MAX_BATCH_SIZE = 4) -- **Prompt Array Length**: When using different prompts, the prompt array length must match the batch_size -- **Memory Requirements**: Larger batches require more GPU memory - -#### Displaying Batch Results - -Use the included `show_batch.py` script to handle multiple images in the response: - -```bash -# The script automatically detects single vs multiple images -curl ... | python show_batch.py -``` - -The script will: -- Save each image with a unique filename -- Automatically open all generated images -- Print status information about the batch - -### Load Test -Before running the load test, update `load_test.py` with your actual endpoint URL and API key. Replace the placeholder values for `api_url` and `api_key` with your deployment's information (lines 43 and 44). - -```bash -python load_test.py --save-all-images --use-varied-prompts --concurrent --num-requests 30 - -πŸš€ Starting Flux Truss API test... -================================================== -🎨 Using 30 varied prompts for load testing -πŸ“ First 3 prompts to be tested: - 1. a beautiful photograph of Mt. Fuji during cherry blossom, photorealistic, high quality - 2. a majestic dragon soaring through a mystical forest, digital art, detailed - 3. a cozy coffee shop interior with warm lighting, people working on laptops, photorealistic - ... and 27 more -πŸš€ Starting concurrent load test with 30 requests, max 5 workers -============================================================ -πŸ“€ Sending request 1/30: 'a beautiful photograph of Mt. Fuji during cherry b...' -Testing Truss API endpoint with prompt: 'a beautiful photograph of Mt. Fuji during cherry blossom, photorealistic, high quality' - -... - -============================================================ -πŸ“Š LOAD TEST SUMMARY -============================================================ -Total requests: 30 -Successful: 30 -Failed: 0 -Success rate: 100.0% -Total time: 74.72 seconds -Average request time: 11.63 seconds -Min request time: 2.91 seconds -Max request time: 12.86 seconds -Throughput: 0.40 requests/second - -================================================== - -πŸ’Ύ Saving 30 successful images... -βœ… Successfully saved 30/30 images to './output' -πŸ“ Opened output directory - -================================================== -``` - -### Advanced Usage - -```python -{ - "prompt": "A beautiful landscape with mountains and a lake, photorealistic, high quality", - "prompt2": "Additional prompt for T5 tokenizer", # Optional, uses prompt if not provided - "negative_prompt": "blurry, low quality, distorted", # Optional - "height": 1024, # Must be multiple of 8 - "width": 1024, # Must be multiple of 8 - "denoising_steps": 50, # Number of denoising steps - "guidance_scale": 3.5, # Classifier-free guidance scale (must be > 1) - "seed": 42, # Random seed for reproducible results - "batch_size": 1, # Number of images to generate - "batch_count": 1, # Number of batches - "num_warmup_runs": 0, # Number of warmup runs - "max_sequence_length": 512, # Max sequence length (up to 512 for flux.1-dev) - "t5_ws_percentage": None, # T5 weight streaming percentage - "transformer_ws_percentage": None # Transformer weight streaming percentage -} -``` - -## Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `prompt` | string/array | required | Text prompt(s) for image generation. Can be a single string or array of strings for different prompts per batch item | -| `prompt2` | string/array | same as prompt | Additional prompt(s) for T5 tokenizer. Can be a single string or array of strings | -| `negative_prompt` | string/array | "" | Negative prompt(s) to avoid certain elements. Can be a single string or array of strings | -| `height` | int | 1024 | Image height (must be multiple of 8) | -| `width` | int | 1024 | Image width (must be multiple of 8) | -| `denoising_steps` | int | 50 | Number of denoising steps | -| `guidance_scale` | float | 3.5 | Classifier-free guidance scale (> 1) | -| `seed` | int | None | Random seed for reproducibility | -| `batch_size` | int | 1 | Number of images per batch | -| `batch_count` | int | 1 | Number of batches | -| `num_warmup_runs` | int | 0 | Number of warmup runs | -| `max_sequence_length` | int | 512 | Maximum sequence length (≀ 512) | -| `t5_ws_percentage` | int | None | T5 weight streaming percentage | -| `transformer_ws_percentage` | int | None | Transformer weight streaming percentage | - -## Response Format - -The model returns a JSON response with the following structure: - -### Single Image Response - -```json -{ - "status": "success", - "data": "base64_encoded_image", - "time": 2.34, - "prompt": "A beautiful landscape with mountains and a lake, photorealistic, high quality", - "negative_prompt": "blurry, low quality, distorted", - "height": 1024, - "width": 1024, - "num_inference_steps": 30, - "guidance_scale": 3.5, - "seed": 42 -} -``` - -### Batch Response (Multiple Images) - -```json -{ - "status": "success", - "data": [ - "base64_encoded_image_1", - "base64_encoded_image_2", - "base64_encoded_image_3", - "base64_encoded_image_4" - ], - "time": 7.23, - "prompt": [ - "A beautiful landscape with mountains and a lake, photorealistic, high quality", - "A futuristic city skyline at sunset, neon lights, cyberpunk style, high quality", - "A cute cat sitting in a garden, soft lighting, detailed, high quality", - "Abstract geometric patterns in vibrant colors, modern art style, high quality" - ], - "negative_prompt": [ - "blurry, low quality, distorted", - "blurry, low quality, distorted", - "blurry, low quality, distorted", - "blurry, low quality, distorted" - ], - "height": 1024, - "width": 1024, - "num_inference_steps": 30, - "guidance_scale": 3.5, - "seed": 42 -} -``` - -## Performance Notes - -The model is optimized with a pre-compiled TensorRT engine for the NVIDIA B200 GPU. -Performance characteristic is described below for the [basic usage](#basic-usage) - - -```text -|------------------|--------------| -| Module | Latency | -|------------------|--------------| -| CLIP | 2.02 ms | -| T5 | 6.43 ms | -| Transformer x 50 | 2361.44 ms | -| VAE-Dec | 11.67 ms | -|------------------|--------------| -| Pipeline | 2382.45 ms | -|------------------|--------------| -``` - -## Model Variants - -This implementation supports the `flux.1-dev` variant with: -- Maximum sequence length: 512 tokens -- Default image dimensions: 1024x1024 -- Optimized for high-quality image generation diff --git a/flux/README.md b/flux/README.md deleted file mode 100644 index 3f7fa0911..000000000 --- a/flux/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# FLUX.1 - -These are trusses for the brand new FLUX.1 schnell and dev model. - -## Deploy FLUX.1 - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd flux -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Make sure you have access to the [FLUX.1 model](https://huggingface.co/black-forest-labs/FLUX.1-schnell) and add a huggingface access token with read access as a Baseten secret named `hf_access_token`. - -With `flux/dev` or `flux/schnell` as your working directory, you can deploy the model with: - -``` -truss push --trusted --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using FLUX.1 through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -## Invoking FLUX.1 - -The output will be a dictionary with a key `data` mapping to a base64 encoded image. It's processed with this script: - -```python -import httpx -import os -import base64 -from PIL import Image -from io import BytesIO - -# Replace the empty string with your model id below -model_id = "" -baseten_api_key = os.environ["BASETEN_API_KEY"] - -# Function used to convert a base64 string to a PIL image -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str))) - -data = { - "prompt": "a little boy looking through a large magical portal, the boy sees a futuristic human civilization in that portal, extremely detailed, trending on artstation, 8k" -} - -# Call model endpoint -res = httpx.post( - f"https://model-{model_id}.api.baseten.co/production/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=data -) - -# Get output image -res = res.json() -output = res.get("data") - -# Convert the base64 model output to an image -img = b64_to_pil(output) -img.save("output_image.jpg") -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://model-{model_id}.api.baseten.co/production/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "A tree in a field under the night sky" - }' -``` - -Again, the model will return a dictionary containing the base64-encoded image, which will need to be decoded and saved. diff --git a/flux/schnell/README.md b/flux/schnell/README.md deleted file mode 100644 index c4f768de4..000000000 --- a/flux/schnell/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# b10‑transfer + torch.compile example (Flux image generation) - -Speed up cold starts for a diffusers Flux pipeline on Baseten by caching PyTorch compilation artifacts with b10‑tcache. This example shows how to: - -- Compile the heavy parts of the pipeline with torch.compile - -- Warm up the model across supported resolutions - -- Load a previously saved compile cache on startup and save it after first warmup - -In this case, we brought the cold start time from 900s to 70s. - -See the full docs on [b10_transfer](https://docs.baseten.co/development/model/b10-transfer). - -## Prerequisites - -Add the cache helper to your requirements in `config.yaml`: - -```yaml -requirements: - - b10-transfer -``` - -## How it works - -Load (and later save) compile cache via b10_transfer: - -```python -from b10_transfer import load_compile_cache, save_compile_cache, OperationStatus - -cache_loaded = load_compile_cache() - -if cache_loaded == OperationStatus.ERROR: - logging.info("Run in eager mode, skipping torch compile") -else: - self.compile() - -if cache_loaded == OperationStatus.DOES_NOT_EXIST: - save_compile_cache() -``` - -Compile the hotspots of the pipeline with Torch Dynamo/TorchInductor: - -```python -self.pipe.transformer = torch.compile( - self.pipe.transformer, mode="max-autotune-no-cudagraphs", dynamic=False -) -self.pipe.vae.decode = torch.compile( - self.pipe.vae.decode, mode="max-autotune-no-cudagraphs", dynamic=False -) -``` - -Warm up with dummy prompts across every resolution you intend to serve (so later requests hit already‑compiled kernels): - -```python -for width, height in [(1024, 1024), (1216, 832), (896, 1152)]: - self.pipe( - prompt="dummy prompt", - prompt_2=None, - guidance_scale=0.0, - max_sequence_length=256, - num_inference_steps=4, - width=width, - height=height, - output_type="pil", - generator=generator, - ) - self.pipe( - prompt="extra dummy prompt", - prompt_2=None, - guidance_scale=0.0, - max_sequence_length=256, - num_inference_steps=4, - width=width, - height=height, - output_type="pil", - generator=generator, - ) -``` - -On subsequent cold starts, load_compile_cache() restores previously compiled artifacts and dramatically reduces compile latency. diff --git a/fotographer/zenctrl/README.md b/fotographer/zenctrl/README.md deleted file mode 100644 index 27bac2687..000000000 --- a/fotographer/zenctrl/README.md +++ /dev/null @@ -1,17 +0,0 @@ -![Header Image](images/banner_1.png) - -# Fotographer AI ZenCtrl - -Image-to-image model for generating in-context product photography. - -Deploy with `truss push --promote` - -Call with `python call.py` after providing `model_id` from deployed model. - -### Example input image - -![Speaker Input](images/speaker-input.png) - -### Example output image - -![Speaker Input](images/speaker-output.png) diff --git a/gemma/gemma-2-27b-it-vllm/README.md b/gemma/gemma-2-27b-it-vllm/README.md deleted file mode 100644 index 093531597..000000000 --- a/gemma/gemma-2-27b-it-vllm/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Gemma 2 27B - -This is a [Truss](https://truss.baseten.co/) for Gemma 2 27B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Gemma 2 27B Instruct. - -## Gemma 2 27B Instruct Implementation - -This implementation of Gemma 2 uses [vLLM](https://github.com/vllm-project/vllm). - -Since Gemma 2 is a gated model, you will also need to provide your Huggingface access token after making sure you have access to [the model](https://huggingface.co/google/gemma-2-27b-it). Please use the [following guide](https://docs.baseten.co/deploy/guides/secrets) to add your Huggingface access token as a secret. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd gemma/gemma-2-27b-it-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `gemma/gemma-2-27b-it-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Gemma 2 27B Instruct API documentation - -This section provides an overview of the Gemma 2 27B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __max_tokens__: The maximum number of output tokens. - -## Example usage - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{"prompt": "what came before, the chicken or the egg?", "max_tokens": 64}' -``` diff --git a/gemma/gemma-2-9b-it-vllm/README.md b/gemma/gemma-2-9b-it-vllm/README.md deleted file mode 100644 index b660f4896..000000000 --- a/gemma/gemma-2-9b-it-vllm/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Gemma 2 9B - -This is a [Truss](https://truss.baseten.co/) for Gemma 2 9B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Gemma 2 9B Instruct. - -## Gemma 2 9B Instruct Implementation - -This implementation of Gemma 2 uses [vLLM](https://github.com/vllm-project/vllm). - -Since Gemma 2 is a gated model, you will also need to provide your Huggingface access token after making sure you have access to [the model](https://huggingface.co/google/gemma-2-9b-it). Please use the [following guide](https://docs.baseten.co/deploy/guides/secrets) to add your Huggingface access token as a secret. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd gemma/gemma-2-9b-it-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `gemma/gemma-2-9b-it-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Gemma 2 9B Instruct API documentation - -This section provides an overview of the Gemma 2 9B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __max_tokens__: The maximum number of output tokens. - -## Example usage - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{"prompt": "what came before, the chicken or the egg?", "max_tokens": 64}' -``` diff --git a/gfp-gan/README.md b/gfp-gan/README.md deleted file mode 100644 index 611703192..000000000 --- a/gfp-gan/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# GFP-GAN Truss - -This is a [Truss](https://truss.baseten.co/) for serving an implementation of TencentARC -[GFPGAN](https://github.com/TencentARC/GFPGAN). GFPGAN is an algorithm for real-world face restoration. -It can be used on old photos of faces to remove blur, and increase clarity and resolution. - -It leverages rich and diverse priors encapsulated in a pretrained face GAN (e.g., StyleGAN2) for -"blind face" restoration. - -## Deploying GFP-GAN - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd gfp-gan-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `gfp-gan-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## GFP-GAN API documentation - -### Input - -The input should be a dictionary with the following key: - -- `image` - the image to be restored, encoded as base64. - -### Output - -The model returns a dictionary containing the base64-encoded restored image: - -- `status` - either `success` or `failed` -- `data` - the restored image, encoded as base64 -- `message` - will contain details in the case of errors - -## Example usage - -```sh -truss predict -d '{"image": "{BASE_64_INPUT}"}' -``` - -You can also invoke this model on Baseten with the following cURL command (just fill in the model version ID and API Key): - -``` -$ curl -X POST https://app.baseten.co/models/{MODEL_VERSION_ID}/predict \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{"image": "{BASE_64_INPUT}"}' -``` diff --git a/grpc/README.md b/grpc/README.md deleted file mode 100644 index 90df3f483..000000000 --- a/grpc/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# gRPC on Baseten - -This example demonstrates how to deploy a gRPC model on Baseten using Truss. - -# Prerequisites - -1. **Install Truss:** - ```bash - pip install --upgrade truss - ``` - -2. **Install Protocol Buffer compiler:** - ```bash - # On macOS - brew install protobuf - - # On Ubuntu/Debian - sudo apt-get install protobuf-compiler - - # On other systems, see: https://protobuf.dev/getting-started/ - ``` - -3. **Install gRPC tools:** - ```bash - pip install grpcio-tools - ``` - -# Steps to Deploy - -### Step 1: Generate Protocol Buffer Code - -Generate the Python code from your `.proto` file: - -```bash -python -m grpc_tools.protoc --python_out=. --grpc_python_out=. --proto_path . example.proto -``` - -### Step 2: Build and Push Docker Image - -Build and push your Docker image to a container registry: - -```bash -docker build -t your-registry/truss-grpc-demo:latest . --platform linux/amd64 -docker push your-registry/truss-grpc-demo:latest -``` - -### Step 3: Configure your Truss - -Update the `config.yaml` file with your model name and Docker image: - -```yaml -model_name: "gRPC Model Example" -base_image: - image: your-registry/truss-grpc-demo:latest -``` - -### Step 4: Deploy with Truss - -Deploy your model using the Truss CLI: - -```bash -truss push --promote -``` - -### Step 5: Invoke the Model - -Update the code in `client.py` to connect to your deployed model. Replace `{MODEL_ID}` with your actual model ID, -and the API_KEY with your Baseten API key: - -Run your client to test the deployed model: - -```bash -python client.py -``` diff --git a/image-segmentation/config.yaml b/image-segmentation/config.yaml deleted file mode 100644 index 6602b1035..000000000 --- a/image-segmentation/config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_name: Image segmentation -python_version: py39 -requirements: -- torchvision==0.9.1 -resources: - cpu: 3000m - memory: 8Gi - use_gpu: false -secrets: {} -spec_version: 2.0 -system_packages: [] diff --git a/image/README.md b/image/README.md new file mode 100644 index 000000000..d43d01bf6 --- /dev/null +++ b/image/README.md @@ -0,0 +1,29 @@ +# Image Models + +Truss configurations for image generation, editing, and segmentation models. Covers Stable Diffusion, Flux, and a variety of specialized image processing pipelines. + +| Directory | Variants | Description | +| --------------------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------- | +| [stable-diffusion](stable-diffusion/) | 18 | Stable Diffusion family including SD 1.x, SDXL, SD 3, turbo, LCM, LoRA, ControlNet, inpainting, TensorRT, and video diffusion | +| [flux](flux/) | 2 | Black Forest Labs Flux models (dev, schnell) | +| [flux-dev-trt-b200](flux-dev-trt-b200/) | 1 | Flux Dev optimized with TensorRT for B200 GPUs | +| [sana](sana/) | 2 | Sana image generation models (600M, 1600M) | +| [comfyui](comfyui/) | 1 | ComfyUI workflow server for node-based image generation | +| [control-net-qrcode](control-net-qrcode/) | 1 | ControlNet QR code art generation | +| [deepfloyd-xl](deepfloyd-xl/) | 1 | DeepFloyd IF XL text-to-image model | +| [fotographer](fotographer/) | 1 | Fotographer AI portrait generation model | +| [gfp-gan](gfp-gan/) | 1 | GFPGAN face restoration and enhancement | +| [ip-adapter](ip-adapter/) | 1 | IP-Adapter for image-prompted generation | +| [magic-animate](magic-animate/) | 1 | MagicAnimate human image animation | +| [playground-v2-aesthetic](playground-v2-aesthetic/) | 1 | Playground v2 aesthetic image generation | +| [segment-anything](segment-anything/) | 1 | Meta SAM universal image segmentation | +| [dis-segmentation](dis-segmentation/) | 1 | Dichotomous image segmentation for high-accuracy cutouts | +| [image-segmentation](image-segmentation/) | 1 | General-purpose image segmentation | + +## Deploying + +Each image model can be deployed to Baseten with: + +```bash +truss push +``` diff --git a/image/comfyui/README.md b/image/comfyui/README.md new file mode 100644 index 000000000..d957e93e2 --- /dev/null +++ b/image/comfyui/README.md @@ -0,0 +1,36 @@ +# ComfyUI Workflow + +Deploy a ComfyUI workflow as a Truss + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "workflow_values": { + "controlnet_image": "https://storage.googleapis.com/logos-bucket-01/baseten_logo.png", + "negative_prompt": "blurry, text, low quality", + "positive_prompt": "An igloo on a snowy day, 4k, hd" + } +}' +``` + +## Configuration highlights + +- Base image: `bolabaseten/comfyui-truss-base:6a7bc35` +- System packages: `ffmpeg, libgl1-mesa-glx` diff --git a/comfyui-truss/config.yaml b/image/comfyui/config.yaml similarity index 89% rename from comfyui-truss/config.yaml rename to image/comfyui/config.yaml index b058b6c92..f24e7d599 100644 --- a/comfyui-truss/config.yaml +++ b/image/comfyui/config.yaml @@ -5,6 +5,7 @@ description: Deploy a ComfyUI workflow as a Truss environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "stabilityai/stable-diffusion-xl-base-1.0" example_model_input: workflow_values: controlnet_image: https://storage.googleapis.com/logos-bucket-01/baseten_logo.png @@ -15,7 +16,7 @@ python_version: py39 requirements: - websocket-client==1.6.4 - accelerate==0.23.0 -- opencv-python +- opencv-python==4.8.1.78 resources: accelerator: A10G cpu: '3' diff --git a/comfyui-truss/data/comfy_ui_workflow.json b/image/comfyui/data/comfy_ui_workflow.json similarity index 100% rename from comfyui-truss/data/comfy_ui_workflow.json rename to image/comfyui/data/comfy_ui_workflow.json diff --git a/comfyui-truss/data/model.json b/image/comfyui/data/model.json similarity index 100% rename from comfyui-truss/data/model.json rename to image/comfyui/data/model.json diff --git a/comfyui-truss/examples/animate-diff/model.json b/image/comfyui/examples/animate-diff/model.json similarity index 100% rename from comfyui-truss/examples/animate-diff/model.json rename to image/comfyui/examples/animate-diff/model.json diff --git a/comfyui-truss/examples/animate-diff/workflow.json b/image/comfyui/examples/animate-diff/workflow.json similarity index 100% rename from comfyui-truss/examples/animate-diff/workflow.json rename to image/comfyui/examples/animate-diff/workflow.json diff --git a/image/comfyui/examples/anime-style-transfer/README.md b/image/comfyui/examples/anime-style-transfer/README.md new file mode 100644 index 000000000..8493cc762 --- /dev/null +++ b/image/comfyui/examples/anime-style-transfer/README.md @@ -0,0 +1,31 @@ +# ComfyUI Anime Pet Style Transfer + +Deploy ComfyUI Anime Pet Style Transfer for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- System packages: `wget, ffmpeg, libgl1-mesa-glx` diff --git a/comfyui-truss/examples/anime-style-transfer/config.yaml b/image/comfyui/examples/anime-style-transfer/config.yaml similarity index 86% rename from comfyui-truss/examples/anime-style-transfer/config.yaml rename to image/comfyui/examples/anime-style-transfer/config.yaml index 04442c3d3..abfc3c94e 100644 --- a/comfyui-truss/examples/anime-style-transfer/config.yaml +++ b/image/comfyui/examples/anime-style-transfer/config.yaml @@ -1,3 +1,4 @@ +description: "ComfyUI Anime Pet Style Transfer for image generation" build_commands: - git clone https://github.com/comfyanonymous/ComfyUI.git - cd ComfyUI && git checkout b1fd26fe9e55163f780bf9e5f56bf9bf5f035c93 && pip install -r requirements.txt @@ -12,13 +13,18 @@ build_commands: - cd ComfyUI/models/loras && wget -O StudioGhibli.Redmond-StdGBRRedmAF-StudioGhibli.safetensors https://huggingface.co/artificialguybr/StudioGhibli.Redmond-V2/resolve/main/StudioGhibli.Redmond-StdGBRRedmAF-StudioGhibli.safetensors environment_variables: {} external_package_dirs: [] -model_metadata: {} +model_metadata: + repo_id: "stabilityai/stable-diffusion-xl-base-1.0" + example_model_input: + workflow_values: + prompt: A cute cat sitting on a windowsill + input_image: https://example.com/image.jpg model_name: ComfyUI Anime Pet Style Transfer python_version: py310 requirements: - websocket-client==1.6.4 - accelerate==0.23.0 - - opencv-python + - opencv-python==4.8.1.78 resources: accelerator: A100 use_gpu: true diff --git a/comfyui-truss/examples/anime-style-transfer/workflow.json b/image/comfyui/examples/anime-style-transfer/workflow.json similarity index 100% rename from comfyui-truss/examples/anime-style-transfer/workflow.json rename to image/comfyui/examples/anime-style-transfer/workflow.json diff --git a/comfyui-truss/examples/sdxl-controlnet/model.json b/image/comfyui/examples/sdxl-controlnet/model.json similarity index 100% rename from comfyui-truss/examples/sdxl-controlnet/model.json rename to image/comfyui/examples/sdxl-controlnet/model.json diff --git a/comfyui-truss/examples/sdxl-controlnet/workflow.json b/image/comfyui/examples/sdxl-controlnet/workflow.json similarity index 100% rename from comfyui-truss/examples/sdxl-controlnet/workflow.json rename to image/comfyui/examples/sdxl-controlnet/workflow.json diff --git a/comfyui-truss/examples/sdxl-with-refiner/model.json b/image/comfyui/examples/sdxl-with-refiner/model.json similarity index 100% rename from comfyui-truss/examples/sdxl-with-refiner/model.json rename to image/comfyui/examples/sdxl-with-refiner/model.json diff --git a/comfyui-truss/examples/sdxl-with-refiner/workflow.json b/image/comfyui/examples/sdxl-with-refiner/workflow.json similarity index 100% rename from comfyui-truss/examples/sdxl-with-refiner/workflow.json rename to image/comfyui/examples/sdxl-with-refiner/workflow.json diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/__init__.py b/image/comfyui/model/__init__.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/__init__.py rename to image/comfyui/model/__init__.py diff --git a/comfyui-truss/model/helpers.py b/image/comfyui/model/helpers.py similarity index 100% rename from comfyui-truss/model/helpers.py rename to image/comfyui/model/helpers.py diff --git a/comfyui-truss/model/model.py b/image/comfyui/model/model.py similarity index 100% rename from comfyui-truss/model/model.py rename to image/comfyui/model/model.py diff --git a/image/control-net-qrcode/README.md b/image/control-net-qrcode/README.md new file mode 100644 index 000000000..b60bc1d15 --- /dev/null +++ b/image/control-net-qrcode/README.md @@ -0,0 +1,32 @@ +# control-net-qrcode + +Deploy control-net-qrcode for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A cubism painting of the Garden of Eaden with animals walking around, Andreas Rocha, matte painting concept art, a detailed matte painting", + "qr_code_content": "https://www.baseten.co" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/control-net-qrcode/config.yaml b/image/control-net-qrcode/config.yaml similarity index 82% rename from control-net-qrcode/config.yaml rename to image/control-net-qrcode/config.yaml index 7744192c2..5e8a55731 100644 --- a/control-net-qrcode/config.yaml +++ b/image/control-net-qrcode/config.yaml @@ -1,6 +1,8 @@ +description: "ControlNet QR Code for artistic QR code image generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "monster-labs/control_v1p_sd15_qrcode_monster" example_model_input: prompt: A cubism painting of the Garden of Eaden with animals walking around, Andreas Rocha, matte painting concept art, a detailed matte painting diff --git a/control-net-qrcode/controlnet_qr_code_results.gif b/image/control-net-qrcode/controlnet_qr_code_results.gif similarity index 100% rename from control-net-qrcode/controlnet_qr_code_results.gif rename to image/control-net-qrcode/controlnet_qr_code_results.gif diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/__init__.py b/image/control-net-qrcode/model/__init__.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/__init__.py rename to image/control-net-qrcode/model/__init__.py diff --git a/control-net-qrcode/model/model.py b/image/control-net-qrcode/model/model.py similarity index 100% rename from control-net-qrcode/model/model.py rename to image/control-net-qrcode/model/model.py diff --git a/control-net-qrcode/twitter_mask.jpeg b/image/control-net-qrcode/twitter_mask.jpeg similarity index 100% rename from control-net-qrcode/twitter_mask.jpeg rename to image/control-net-qrcode/twitter_mask.jpeg diff --git a/control-net-qrcode/twitter_output.jpg b/image/control-net-qrcode/twitter_output.jpg similarity index 100% rename from control-net-qrcode/twitter_output.jpg rename to image/control-net-qrcode/twitter_output.jpg diff --git a/image/deepfloyd-xl/README.md b/image/deepfloyd-xl/README.md new file mode 100644 index 000000000..7ed98c40a --- /dev/null +++ b/image/deepfloyd-xl/README.md @@ -0,0 +1,33 @@ +# Deepfloyd XL + +Generate original images from text prompts. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/deepfloyd-xl/config.yaml b/image/deepfloyd-xl/config.yaml similarity index 62% rename from deepfloyd-xl/config.yaml rename to image/deepfloyd-xl/config.yaml index f61ffd196..d9c410018 100644 --- a/deepfloyd-xl/config.yaml +++ b/image/deepfloyd-xl/config.yaml @@ -2,22 +2,25 @@ description: Generate original images from text prompts. environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "DeepFloyd/IF-I-XL-v1.0" avatar_url: https://cdn.baseten.co/production/static/explore/deep-floyd.png cover_image_url: https://cdn.baseten.co/production/static/explore/deepfloyd-cover.png + example_model_input: + prompt: A photo of an astronaut riding a horse tags: - image-generation model_name: Deepfloyd XL python_version: py39 requirements: -- diffusers -- transformers -- torch -- scipy -- accelerate -- pillow -- bitsandbytes -- sentencepiece -- huggingface_hub +- diffusers==0.24.0 +- transformers==4.36.0 +- torch==2.1.0 +- scipy==1.11.4 +- accelerate==0.25.0 +- pillow==10.1.0 +- bitsandbytes==0.41.3 +- sentencepiece==0.1.99 +- huggingface_hub==0.19.4 resources: accelerator: A10G cpu: '3' @@ -25,5 +28,5 @@ resources: use_gpu: true secrets: hf_access_token: ENTER HF API KEY HERE -spec_version: 2.0 +spec_version: "2.0" system_packages: [] diff --git a/flux/dev/model/__init__.py b/image/deepfloyd-xl/model/__init__.py similarity index 100% rename from flux/dev/model/__init__.py rename to image/deepfloyd-xl/model/__init__.py diff --git a/deepfloyd-xl/model/model.py b/image/deepfloyd-xl/model/model.py similarity index 100% rename from deepfloyd-xl/model/model.py rename to image/deepfloyd-xl/model/model.py diff --git a/image/dis-segmentation/README.md b/image/dis-segmentation/README.md new file mode 100644 index 000000000..362b2c20c --- /dev/null +++ b/image/dis-segmentation/README.md @@ -0,0 +1,31 @@ +# DIS Segmentation + +Deploy DIS Segmentation for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "input_image": "" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/dis-segmentation/config.yaml b/image/dis-segmentation/config.yaml similarity index 80% rename from dis-segmentation/config.yaml rename to image/dis-segmentation/config.yaml index 86c3e7e79..549b0a66d 100644 --- a/dis-segmentation/config.yaml +++ b/image/dis-segmentation/config.yaml @@ -1,6 +1,8 @@ +description: "DIS for high-accuracy image segmentation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "ECCV2022/dis-background-removal" example_model_input: input_image: model_name: DIS Segmentation diff --git a/flux/schnell/model/__init__.py b/image/dis-segmentation/model/__init__.py similarity index 100% rename from flux/schnell/model/__init__.py rename to image/dis-segmentation/model/__init__.py diff --git a/dis-segmentation/model/clone_repo_helper.py b/image/dis-segmentation/model/clone_repo_helper.py similarity index 100% rename from dis-segmentation/model/clone_repo_helper.py rename to image/dis-segmentation/model/clone_repo_helper.py diff --git a/dis-segmentation/model/helpers.py b/image/dis-segmentation/model/helpers.py similarity index 100% rename from dis-segmentation/model/helpers.py rename to image/dis-segmentation/model/helpers.py diff --git a/dis-segmentation/model/model.py b/image/dis-segmentation/model/model.py similarity index 100% rename from dis-segmentation/model/model.py rename to image/dis-segmentation/model/model.py diff --git a/image/flux-dev-trt-b200/README.md b/image/flux-dev-trt-b200/README.md new file mode 100644 index 000000000..98759196b --- /dev/null +++ b/image/flux-dev-trt-b200/README.md @@ -0,0 +1,34 @@ +# Flux 1.0 Dev - TensorRT + +Generate high-quality images from text prompts using Black Forest Labs's Flux model with TensorRT optimization. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | B200 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Base image: `nvcr.io/nvidia/pytorch:25.06-py3` +- Predict concurrency: **1** +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/flux.1-dev-trt-b200/config.yaml b/image/flux-dev-trt-b200/config.yaml similarity index 76% rename from flux.1-dev-trt-b200/config.yaml rename to image/flux-dev-trt-b200/config.yaml index d2b6b2e2f..c72a45919 100644 --- a/flux.1-dev-trt-b200/config.yaml +++ b/image/flux-dev-trt-b200/config.yaml @@ -2,6 +2,10 @@ base_image: image: nvcr.io/nvidia/pytorch:25.06-py3 description: Generate high-quality images from text prompts using Black Forest Labs's Flux model with TensorRT optimization. external_package_dirs: [] +model_metadata: + repo_id: "black-forest-labs/FLUX.1-dev" + example_model_input: + prompt: A photo of an astronaut riding a horse model_name: Flux 1.0 Dev - TensorRT requirements_file: ./requirements.txt resources: diff --git a/flux.1-dev-trt-b200/load_test.py b/image/flux-dev-trt-b200/load_test.py similarity index 100% rename from flux.1-dev-trt-b200/load_test.py rename to image/flux-dev-trt-b200/load_test.py diff --git a/gemma/gemma-2-27b-it-vllm/model/__init__.py b/image/flux-dev-trt-b200/model/__init__.py similarity index 100% rename from gemma/gemma-2-27b-it-vllm/model/__init__.py rename to image/flux-dev-trt-b200/model/__init__.py diff --git a/gemma/gemma-2-9b-it-vllm/model/__init__.py b/image/flux-dev-trt-b200/model/demo_diffusion/__init__.py similarity index 100% rename from gemma/gemma-2-9b-it-vllm/model/__init__.py rename to image/flux-dev-trt-b200/model/demo_diffusion/__init__.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/dd_argparse.py b/image/flux-dev-trt-b200/model/demo_diffusion/dd_argparse.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/dd_argparse.py rename to image/flux-dev-trt-b200/model/demo_diffusion/dd_argparse.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/dynamic_import.py b/image/flux-dev-trt-b200/model/demo_diffusion/dynamic_import.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/dynamic_import.py rename to image/flux-dev-trt-b200/model/demo_diffusion/dynamic_import.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/engine.py b/image/flux-dev-trt-b200/model/demo_diffusion/engine.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/engine.py rename to image/flux-dev-trt-b200/model/demo_diffusion/engine.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/image/__init__.py b/image/flux-dev-trt-b200/model/demo_diffusion/image/__init__.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/image/__init__.py rename to image/flux-dev-trt-b200/model/demo_diffusion/image/__init__.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/image/load.py b/image/flux-dev-trt-b200/model/demo_diffusion/image/load.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/image/load.py rename to image/flux-dev-trt-b200/model/demo_diffusion/image/load.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/image/resize.py b/image/flux-dev-trt-b200/model/demo_diffusion/image/resize.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/image/resize.py rename to image/flux-dev-trt-b200/model/demo_diffusion/image/resize.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/image/video.py b/image/flux-dev-trt-b200/model/demo_diffusion/image/video.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/image/video.py rename to image/flux-dev-trt-b200/model/demo_diffusion/image/video.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/__init__.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/__init__.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/__init__.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/__init__.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/base_model.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/base_model.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/base_model.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/base_model.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/clip.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/clip.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/clip.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/clip.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/diffusion_transformer.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/diffusion_transformer.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/diffusion_transformer.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/diffusion_transformer.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/gan.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/gan.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/gan.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/gan.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/load.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/load.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/load.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/load.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/lora.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/lora.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/lora.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/lora.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/optimizer.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/optimizer.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/optimizer.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/optimizer.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/scheduler.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/scheduler.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/scheduler.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/scheduler.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/t5.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/t5.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/t5.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/t5.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/tokenizer.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/tokenizer.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/tokenizer.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/tokenizer.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/unet.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/unet.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/unet.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/unet.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/model/vae.py b/image/flux-dev-trt-b200/model/demo_diffusion/model/vae.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/model/vae.py rename to image/flux-dev-trt-b200/model/demo_diffusion/model/vae.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/path/__init__.py b/image/flux-dev-trt-b200/model/demo_diffusion/path/__init__.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/path/__init__.py rename to image/flux-dev-trt-b200/model/demo_diffusion/path/__init__.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/path/dd_path.py b/image/flux-dev-trt-b200/model/demo_diffusion/path/dd_path.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/path/dd_path.py rename to image/flux-dev-trt-b200/model/demo_diffusion/path/dd_path.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/path/resolve_path.py b/image/flux-dev-trt-b200/model/demo_diffusion/path/resolve_path.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/path/resolve_path.py rename to image/flux-dev-trt-b200/model/demo_diffusion/path/resolve_path.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/__init__.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/__init__.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/__init__.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/__init__.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/calibrate.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/calibrate.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/calibrate.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/calibrate.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/diffusion_pipeline.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/diffusion_pipeline.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/diffusion_pipeline.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/diffusion_pipeline.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/flux_pipeline.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/flux_pipeline.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/flux_pipeline.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/flux_pipeline.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/model_memory_manager.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/model_memory_manager.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/model_memory_manager.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/model_memory_manager.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_cascade_pipeline.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_cascade_pipeline.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_cascade_pipeline.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_cascade_pipeline.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_35_pipeline.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_35_pipeline.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_35_pipeline.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_35_pipeline.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_3_pipeline.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_3_pipeline.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_3_pipeline.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_3_pipeline.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_pipeline.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_pipeline.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_pipeline.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_diffusion_pipeline.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_video_diffusion_pipeline.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_video_diffusion_pipeline.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/stable_video_diffusion_pipeline.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/stable_video_diffusion_pipeline.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/pipeline/type.py b/image/flux-dev-trt-b200/model/demo_diffusion/pipeline/type.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/pipeline/type.py rename to image/flux-dev-trt-b200/model/demo_diffusion/pipeline/type.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/utils_modelopt.py b/image/flux-dev-trt-b200/model/demo_diffusion/utils_modelopt.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/utils_modelopt.py rename to image/flux-dev-trt-b200/model/demo_diffusion/utils_modelopt.py diff --git a/gfp-gan/model/__init__.py b/image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/__init__.py similarity index 100% rename from gfp-gan/model/__init__.py rename to image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/__init__.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/mmdit.py b/image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/mmdit.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/mmdit.py rename to image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/mmdit.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/other_impls.py b/image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/other_impls.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/other_impls.py rename to image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/other_impls.py diff --git a/flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/sd3_impls.py b/image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/sd3_impls.py similarity index 100% rename from flux.1-dev-trt-b200/model/demo_diffusion/utils_sd3/sd3_impls.py rename to image/flux-dev-trt-b200/model/demo_diffusion/utils_sd3/sd3_impls.py diff --git a/flux.1-dev-trt-b200/model/model.py b/image/flux-dev-trt-b200/model/model.py similarity index 100% rename from flux.1-dev-trt-b200/model/model.py rename to image/flux-dev-trt-b200/model/model.py diff --git a/flux.1-dev-trt-b200/requirements.txt b/image/flux-dev-trt-b200/requirements.txt similarity index 100% rename from flux.1-dev-trt-b200/requirements.txt rename to image/flux-dev-trt-b200/requirements.txt diff --git a/flux.1-dev-trt-b200/show.py b/image/flux-dev-trt-b200/show.py similarity index 100% rename from flux.1-dev-trt-b200/show.py rename to image/flux-dev-trt-b200/show.py diff --git a/flux.1-dev-trt-b200/show_batch.py b/image/flux-dev-trt-b200/show_batch.py similarity index 100% rename from flux.1-dev-trt-b200/show_batch.py rename to image/flux-dev-trt-b200/show_batch.py diff --git a/image/flux/README.md b/image/flux/README.md new file mode 100644 index 000000000..bcfec3909 --- /dev/null +++ b/image/flux/README.md @@ -0,0 +1,46 @@ +# FLUX.1 + +Deploy [FLUX.1](https://huggingface.co/black-forest-labs/FLUX.1-schnell) image generation models on Baseten. This directory contains two variants: + +| Variant | Path | GPU | HuggingFace | +|---------|------|-----|-------------| +| FLUX.1 Dev | [`dev/`](dev/) | H100 40GB | [black-forest-labs/FLUX.1-dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) | +| FLUX.1 Schnell | [`schnell/`](schnell/) | H100 40GB | [black-forest-labs/FLUX.1-schnell](https://huggingface.co/black-forest-labs/FLUX.1-schnell) | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push image/flux/dev +# or +truss push image/flux/schnell +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "black forest gateau cake spelling out the words FLUX DEV, tasty, food photography, dynamic shot"}' +``` + +The model returns a dictionary with a `data` key containing a base64-encoded image: + +```python +import requests +import base64 +from PIL import Image +from io import BytesIO + +res = requests.post( + "https://model-.api.baseten.co/predict", + headers={"Authorization": "Api-Key YOUR_BASETEN_API_KEY"}, + json={"prompt": "A tree in a field under the night sky"}, +) + +output = res.json()["data"] +img = Image.open(BytesIO(base64.b64decode(output))) +img.save("output_image.jpg") +``` diff --git a/image/flux/dev/README.md b/image/flux/dev/README.md new file mode 100644 index 000000000..8f0eeeadd --- /dev/null +++ b/image/flux/dev/README.md @@ -0,0 +1,34 @@ +# Flux.1-dev + +Deploy [black-forest-labs/FLUX.1-dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Model | [black-forest-labs/FLUX.1-dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | H100_40GB | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "black forest gateau cake spelling out the words \"FLUX DEV\", tasty, food photography, dynamic shot" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/flux/dev/config.yaml b/image/flux/dev/config.yaml similarity index 76% rename from flux/dev/config.yaml rename to image/flux/dev/config.yaml index 61e1a7d40..c7bb29409 100644 --- a/flux/dev/config.yaml +++ b/image/flux/dev/config.yaml @@ -1,3 +1,4 @@ +description: "black-forest-labs/FLUX.1-dev for image generation" external_package_dirs: [] model_metadata: example_model_input: {"prompt": 'black forest gateau cake spelling out the words "FLUX DEV", tasty, food photography, dynamic shot'} @@ -6,10 +7,10 @@ model_name: Flux.1-dev python_version: py311 requirements: - git+https://github.com/huggingface/diffusers.git@fc6a91e3834c35e57b398ad1c0d99f6f83557e04 - - transformers - - accelerate - - sentencepiece - - protobuf + - transformers==4.36.0 + - accelerate==0.25.0 + - sentencepiece==0.1.99 + - protobuf==4.25.1 resources: accelerator: H100_40GB use_gpu: true diff --git a/image-segmentation/model/__init__.py b/image/flux/dev/model/__init__.py similarity index 100% rename from image-segmentation/model/__init__.py rename to image/flux/dev/model/__init__.py diff --git a/flux/dev/model/model.py b/image/flux/dev/model/model.py similarity index 100% rename from flux/dev/model/model.py rename to image/flux/dev/model/model.py diff --git a/image/flux/schnell/README.md b/image/flux/schnell/README.md new file mode 100644 index 000000000..a7cf3d25f --- /dev/null +++ b/image/flux/schnell/README.md @@ -0,0 +1,34 @@ +# Flux.1-schnell + +Deploy [black-forest-labs/FLUX.1-schnell](https://huggingface.co/black-forest-labs/FLUX.1-schnell) for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Model | [black-forest-labs/FLUX.1-schnell](https://huggingface.co/black-forest-labs/FLUX.1-schnell) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | H100_40GB | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "black forest gateau cake spelling out the words \"FLUX SCHNELL\", tasty, food photography, dynamic shot" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/flux/schnell/config.yaml b/image/flux/schnell/config.yaml similarity index 73% rename from flux/schnell/config.yaml rename to image/flux/schnell/config.yaml index 0e19a616f..7a01e68c9 100644 --- a/flux/schnell/config.yaml +++ b/image/flux/schnell/config.yaml @@ -1,3 +1,4 @@ +description: "black-forest-labs/FLUX.1-schnell for image generation" external_package_dirs: [] model_metadata: example_model_input: {"prompt": 'black forest gateau cake spelling out the words "FLUX SCHNELL", tasty, food photography, dynamic shot'} @@ -6,11 +7,11 @@ model_name: Flux.1-schnell python_version: py311 requirements: - git+https://github.com/huggingface/diffusers.git@fc6a91e3834c35e57b398ad1c0d99f6f83557e04 - - transformers - - accelerate - - sentencepiece - - protobuf - - b10-transfer + - transformers==4.36.0 + - accelerate==0.25.0 + - sentencepiece==0.1.99 + - protobuf==4.25.1 + - b10-transfer==0.0.5 resources: accelerator: H100_40GB use_gpu: true diff --git a/ip-adapter/model/__init__.py b/image/flux/schnell/model/__init__.py similarity index 100% rename from ip-adapter/model/__init__.py rename to image/flux/schnell/model/__init__.py diff --git a/flux/schnell/model/model.py b/image/flux/schnell/model/model.py similarity index 100% rename from flux/schnell/model/model.py rename to image/flux/schnell/model/model.py diff --git a/image/fotographer/zenctrl/README.md b/image/fotographer/zenctrl/README.md new file mode 100644 index 000000000..5ed287a46 --- /dev/null +++ b/image/fotographer/zenctrl/README.md @@ -0,0 +1,34 @@ +# ZenCtrl + +Deploy ZenCtrl for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Docker Server | +| GPU | H100 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/generate \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Base image: `fotographerai/zenctrlstage:latest` +- Predict concurrency: **8** +- Environment variables: `PORT` diff --git a/fotographer/zenctrl/call.py b/image/fotographer/zenctrl/call.py similarity index 100% rename from fotographer/zenctrl/call.py rename to image/fotographer/zenctrl/call.py diff --git a/fotographer/zenctrl/config.yaml b/image/fotographer/zenctrl/config.yaml similarity index 68% rename from fotographer/zenctrl/config.yaml rename to image/fotographer/zenctrl/config.yaml index f199b2185..ec0a19e82 100644 --- a/fotographer/zenctrl/config.yaml +++ b/image/fotographer/zenctrl/config.yaml @@ -1,6 +1,12 @@ +description: "ZenCtrl for image generation" base_image: image: fotographerai/zenctrlstage:latest -model_metadata: {} +model_metadata: + repo_id: "Kijai/ZenCtrl-Flux-comfy" + example_model_input: + image: "" + prompt: A man holding a camera facing the objective + steps: 10 docker_server: start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python -m uvicorn app:app --host 0.0.0.0 --port 8000 --log-level debug" readiness_endpoint: /health diff --git a/fotographer/zenctrl/images/banner_1.png b/image/fotographer/zenctrl/images/banner_1.png similarity index 100% rename from fotographer/zenctrl/images/banner_1.png rename to image/fotographer/zenctrl/images/banner_1.png diff --git a/fotographer/zenctrl/images/camera.png b/image/fotographer/zenctrl/images/camera.png similarity index 100% rename from fotographer/zenctrl/images/camera.png rename to image/fotographer/zenctrl/images/camera.png diff --git a/fotographer/zenctrl/images/speaker-input.png b/image/fotographer/zenctrl/images/speaker-input.png similarity index 100% rename from fotographer/zenctrl/images/speaker-input.png rename to image/fotographer/zenctrl/images/speaker-input.png diff --git a/fotographer/zenctrl/images/speaker-output.png b/image/fotographer/zenctrl/images/speaker-output.png similarity index 100% rename from fotographer/zenctrl/images/speaker-output.png rename to image/fotographer/zenctrl/images/speaker-output.png diff --git a/fotographer/zenctrl/requirements.txt b/image/fotographer/zenctrl/requirements.txt similarity index 100% rename from fotographer/zenctrl/requirements.txt rename to image/fotographer/zenctrl/requirements.txt diff --git a/whisper/whisper-truss/LICENSE b/image/gfp-gan/LICENSE similarity index 100% rename from whisper/whisper-truss/LICENSE rename to image/gfp-gan/LICENSE diff --git a/image/gfp-gan/README.md b/image/gfp-gan/README.md new file mode 100644 index 000000000..8a5ef5398 --- /dev/null +++ b/image/gfp-gan/README.md @@ -0,0 +1,31 @@ +# GFP-GAN + +Restore photos with this image-to-image model. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | GPU (unspecified) | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/gfp-gan/config.yaml b/image/gfp-gan/config.yaml similarity index 90% rename from gfp-gan/config.yaml rename to image/gfp-gan/config.yaml index e047715ee..86d93b6ad 100644 --- a/gfp-gan/config.yaml +++ b/image/gfp-gan/config.yaml @@ -7,8 +7,11 @@ external_data: url: https://baseten-public.s3.us-west-2.amazonaws.com/models/gfp-gan/GFPGANv1.3.pth external_package_dirs: [] model_metadata: + repo_id: "TencentARC/gfpgan" avatar_url: https://cdn.baseten.co/production/static/explore/tencent.png cover_image_url: https://cdn.baseten.co/production/static/explore/gfp-gan.png + example_model_input: + image: "" example_model_input_file: input.json tags: - image-restoration diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/__init__ copy.py b/image/gfp-gan/data/.gitkeep similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/__init__ copy.py rename to image/gfp-gan/data/.gitkeep diff --git a/gfp-gan/input.json b/image/gfp-gan/input.json similarity index 100% rename from gfp-gan/input.json rename to image/gfp-gan/input.json diff --git a/jsonformatter/model/__init__.py b/image/gfp-gan/model/__init__.py similarity index 100% rename from jsonformatter/model/__init__.py rename to image/gfp-gan/model/__init__.py diff --git a/gfp-gan/model/model.py b/image/gfp-gan/model/model.py similarity index 100% rename from gfp-gan/model/model.py rename to image/gfp-gan/model/model.py diff --git a/image/image-segmentation/README.md b/image/image-segmentation/README.md new file mode 100644 index 000000000..976ed76aa --- /dev/null +++ b/image/image-segmentation/README.md @@ -0,0 +1,31 @@ +# Image segmentation + +Deploy Image segmentation for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | CPU | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/image/image-segmentation/config.yaml b/image/image-segmentation/config.yaml new file mode 100644 index 000000000..97e572e75 --- /dev/null +++ b/image/image-segmentation/config.yaml @@ -0,0 +1,19 @@ +description: "Image segmentation for image generation" +environment_variables: {} +external_package_dirs: [] +model_metadata: + repo_id: "nvidia/segformer-b0-finetuned-ade-512-512" + example_model_input: + instances: + - image_url: https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg +model_name: Image segmentation +python_version: py39 +requirements: +- torchvision==0.9.1 +resources: + cpu: 3000m + memory: 8Gi + use_gpu: false +secrets: {} +spec_version: "2.0" +system_packages: [] diff --git a/kokoro/model/__init__.py b/image/image-segmentation/model/__init__.py similarity index 100% rename from kokoro/model/__init__.py rename to image/image-segmentation/model/__init__.py diff --git a/image-segmentation/model/model.py b/image/image-segmentation/model/model.py similarity index 100% rename from image-segmentation/model/model.py rename to image/image-segmentation/model/model.py diff --git a/image/ip-adapter/README.md b/image/ip-adapter/README.md new file mode 100644 index 000000000..0c6c1066f --- /dev/null +++ b/image/ip-adapter/README.md @@ -0,0 +1,31 @@ +# IP Adapter + +Deploy IP Adapter for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/ip-adapter/config.yaml b/image/ip-adapter/config.yaml similarity index 64% rename from ip-adapter/config.yaml rename to image/ip-adapter/config.yaml index e3434bf26..8d39291ef 100644 --- a/ip-adapter/config.yaml +++ b/image/ip-adapter/config.yaml @@ -1,5 +1,10 @@ +description: "IP Adapter for image generation" environment_variables: {} external_package_dirs: [] +model_metadata: + repo_id: "h94/IP-Adapter" + example_model_input: + image: "" model_name: IP Adapter python_version: py311 requirements: diff --git a/layoutlm-document-qa/model/__init__.py b/image/ip-adapter/model/__init__.py similarity index 100% rename from layoutlm-document-qa/model/__init__.py rename to image/ip-adapter/model/__init__.py diff --git a/ip-adapter/model/model.py b/image/ip-adapter/model/model.py similarity index 100% rename from ip-adapter/model/model.py rename to image/ip-adapter/model/model.py diff --git a/image/magic-animate/README.md b/image/magic-animate/README.md new file mode 100644 index 000000000..a1723d1aa --- /dev/null +++ b/image/magic-animate/README.md @@ -0,0 +1,35 @@ +# Magic Animate + +Deploy Magic Animate for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "guidance_scale": 7.5, + "motion_sequence": "", + "reference_image": "", + "seed": 1, + "steps": 10 +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/magic-animate/config.yaml b/image/magic-animate/config.yaml similarity index 87% rename from magic-animate/config.yaml rename to image/magic-animate/config.yaml index f2c414ef1..e3b649a04 100644 --- a/magic-animate/config.yaml +++ b/image/magic-animate/config.yaml @@ -1,6 +1,8 @@ +description: "Magic Animate for video animation from images" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "zcxu-eric/MagicAnimate" example_model_input: guidance_scale: 7.5 motion_sequence: diff --git a/llama/llama-2-13b-chat/model/__init__.py b/image/magic-animate/model/__init__.py similarity index 100% rename from llama/llama-2-13b-chat/model/__init__.py rename to image/magic-animate/model/__init__.py diff --git a/magic-animate/model/configs/inference/inference.yaml b/image/magic-animate/model/configs/inference/inference.yaml similarity index 100% rename from magic-animate/model/configs/inference/inference.yaml rename to image/magic-animate/model/configs/inference/inference.yaml diff --git a/magic-animate/model/configs/prompts/animation.yaml b/image/magic-animate/model/configs/prompts/animation.yaml similarity index 100% rename from magic-animate/model/configs/prompts/animation.yaml rename to image/magic-animate/model/configs/prompts/animation.yaml diff --git a/magic-animate/model/demo/animate.py b/image/magic-animate/model/demo/animate.py similarity index 100% rename from magic-animate/model/demo/animate.py rename to image/magic-animate/model/demo/animate.py diff --git a/magic-animate/model/magicanimate/models/appearance_encoder.py b/image/magic-animate/model/magicanimate/models/appearance_encoder.py similarity index 100% rename from magic-animate/model/magicanimate/models/appearance_encoder.py rename to image/magic-animate/model/magicanimate/models/appearance_encoder.py diff --git a/magic-animate/model/magicanimate/models/attention.py b/image/magic-animate/model/magicanimate/models/attention.py similarity index 100% rename from magic-animate/model/magicanimate/models/attention.py rename to image/magic-animate/model/magicanimate/models/attention.py diff --git a/magic-animate/model/magicanimate/models/controlnet.py b/image/magic-animate/model/magicanimate/models/controlnet.py similarity index 100% rename from magic-animate/model/magicanimate/models/controlnet.py rename to image/magic-animate/model/magicanimate/models/controlnet.py diff --git a/magic-animate/model/magicanimate/models/embeddings.py b/image/magic-animate/model/magicanimate/models/embeddings.py similarity index 100% rename from magic-animate/model/magicanimate/models/embeddings.py rename to image/magic-animate/model/magicanimate/models/embeddings.py diff --git a/magic-animate/model/magicanimate/models/motion_module.py b/image/magic-animate/model/magicanimate/models/motion_module.py similarity index 100% rename from magic-animate/model/magicanimate/models/motion_module.py rename to image/magic-animate/model/magicanimate/models/motion_module.py diff --git a/magic-animate/model/magicanimate/models/mutual_self_attention.py b/image/magic-animate/model/magicanimate/models/mutual_self_attention.py similarity index 100% rename from magic-animate/model/magicanimate/models/mutual_self_attention.py rename to image/magic-animate/model/magicanimate/models/mutual_self_attention.py diff --git a/magic-animate/model/magicanimate/models/orig_attention.py b/image/magic-animate/model/magicanimate/models/orig_attention.py similarity index 100% rename from magic-animate/model/magicanimate/models/orig_attention.py rename to image/magic-animate/model/magicanimate/models/orig_attention.py diff --git a/magic-animate/model/magicanimate/models/resnet.py b/image/magic-animate/model/magicanimate/models/resnet.py similarity index 100% rename from magic-animate/model/magicanimate/models/resnet.py rename to image/magic-animate/model/magicanimate/models/resnet.py diff --git a/magic-animate/model/magicanimate/models/stable_diffusion_controlnet_reference.py b/image/magic-animate/model/magicanimate/models/stable_diffusion_controlnet_reference.py similarity index 100% rename from magic-animate/model/magicanimate/models/stable_diffusion_controlnet_reference.py rename to image/magic-animate/model/magicanimate/models/stable_diffusion_controlnet_reference.py diff --git a/magic-animate/model/magicanimate/models/unet.py b/image/magic-animate/model/magicanimate/models/unet.py similarity index 100% rename from magic-animate/model/magicanimate/models/unet.py rename to image/magic-animate/model/magicanimate/models/unet.py diff --git a/magic-animate/model/magicanimate/models/unet_3d_blocks.py b/image/magic-animate/model/magicanimate/models/unet_3d_blocks.py similarity index 100% rename from magic-animate/model/magicanimate/models/unet_3d_blocks.py rename to image/magic-animate/model/magicanimate/models/unet_3d_blocks.py diff --git a/magic-animate/model/magicanimate/models/unet_controlnet.py b/image/magic-animate/model/magicanimate/models/unet_controlnet.py similarity index 100% rename from magic-animate/model/magicanimate/models/unet_controlnet.py rename to image/magic-animate/model/magicanimate/models/unet_controlnet.py diff --git a/magic-animate/model/magicanimate/pipelines/animation.py b/image/magic-animate/model/magicanimate/pipelines/animation.py similarity index 100% rename from magic-animate/model/magicanimate/pipelines/animation.py rename to image/magic-animate/model/magicanimate/pipelines/animation.py diff --git a/magic-animate/model/magicanimate/pipelines/context.py b/image/magic-animate/model/magicanimate/pipelines/context.py similarity index 100% rename from magic-animate/model/magicanimate/pipelines/context.py rename to image/magic-animate/model/magicanimate/pipelines/context.py diff --git a/magic-animate/model/magicanimate/pipelines/pipeline_animation.py b/image/magic-animate/model/magicanimate/pipelines/pipeline_animation.py similarity index 100% rename from magic-animate/model/magicanimate/pipelines/pipeline_animation.py rename to image/magic-animate/model/magicanimate/pipelines/pipeline_animation.py diff --git a/magic-animate/model/magicanimate/utils/dist_tools.py b/image/magic-animate/model/magicanimate/utils/dist_tools.py similarity index 100% rename from magic-animate/model/magicanimate/utils/dist_tools.py rename to image/magic-animate/model/magicanimate/utils/dist_tools.py diff --git a/magic-animate/model/magicanimate/utils/util.py b/image/magic-animate/model/magicanimate/utils/util.py similarity index 100% rename from magic-animate/model/magicanimate/utils/util.py rename to image/magic-animate/model/magicanimate/utils/util.py diff --git a/magic-animate/model/magicanimate/utils/videoreader.py b/image/magic-animate/model/magicanimate/utils/videoreader.py similarity index 100% rename from magic-animate/model/magicanimate/utils/videoreader.py rename to image/magic-animate/model/magicanimate/utils/videoreader.py diff --git a/magic-animate/model/model.py b/image/magic-animate/model/model.py similarity index 100% rename from magic-animate/model/model.py rename to image/magic-animate/model/model.py diff --git a/image/playground-v2-aesthetic/README.md b/image/playground-v2-aesthetic/README.md new file mode 100644 index 000000000..979cf58a8 --- /dev/null +++ b/image/playground-v2-aesthetic/README.md @@ -0,0 +1,33 @@ +# Playground V2 Aesthetic + +Generate original images from text prompts. + +| Property | Value | +|----------|-------| +| Model | [playgroundai/playground-v2-1024px-aesthetic](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "num_inference_steps": 50, + "prompt": "A scenic mountain landscape" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/playground-v2-aesthetic/config.yaml b/image/playground-v2-aesthetic/config.yaml similarity index 100% rename from playground-v2-aesthetic/config.yaml rename to image/playground-v2-aesthetic/config.yaml diff --git a/llama/llama-2-13b/model/__init__.py b/image/playground-v2-aesthetic/model/__init__.py similarity index 100% rename from llama/llama-2-13b/model/__init__.py rename to image/playground-v2-aesthetic/model/__init__.py diff --git a/playground-v2-aesthetic/model/model.py b/image/playground-v2-aesthetic/model/model.py similarity index 100% rename from playground-v2-aesthetic/model/model.py rename to image/playground-v2-aesthetic/model/model.py diff --git a/playground-v2-aesthetic/show.py b/image/playground-v2-aesthetic/show.py similarity index 100% rename from playground-v2-aesthetic/show.py rename to image/playground-v2-aesthetic/show.py diff --git a/image/sana/sana-1600m/README.md b/image/sana/sana-1600m/README.md new file mode 100644 index 000000000..539ead590 --- /dev/null +++ b/image/sana/sana-1600m/README.md @@ -0,0 +1,40 @@ +# Sana 1600M + +Deploy Sana 1600M for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | H100_40GB | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "a photo of an astronaut riding a horse on mars", + "height": 1024, + "width": 1024, + "guidance_scale": 5.0, + "pag_guidance_scale": 2.0, + "num_inference_steps": 18, + "seed": 4096 +}' +``` + +## Configuration highlights + +- Base image: `alphatozeta/cuda-python:12.1.1-cudnn8-devel-ubuntu22.04` +- System packages: `ffmpeg, libsm6, libxext6, python3.10-venv` diff --git a/sana/sana_1600M/config.yaml b/image/sana/sana-1600m/config.yaml similarity index 86% rename from sana/sana_1600M/config.yaml rename to image/sana/sana-1600m/config.yaml index b3cfbf751..1e1bf8e9c 100644 --- a/sana/sana_1600M/config.yaml +++ b/image/sana/sana-1600m/config.yaml @@ -1,9 +1,11 @@ +description: "Sana 1600M for image generation" build_commands: [] base_image: image: alphatozeta/cuda-python:12.1.1-cudnn8-devel-ubuntu22.04 environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "Efficient-Large-Model/Sana_1600M_1024px_MultiLing_diffusers" example_model_input: { "prompt": "a photo of an astronaut riding a horse on mars", "height": 1024, diff --git a/llama/llama-2-70b-chat/model/__init__.py b/image/sana/sana-1600m/model/__init__.py similarity index 100% rename from llama/llama-2-70b-chat/model/__init__.py rename to image/sana/sana-1600m/model/__init__.py diff --git a/sana/sana_1600M/model/model.py b/image/sana/sana-1600m/model/model.py similarity index 100% rename from sana/sana_1600M/model/model.py rename to image/sana/sana-1600m/model/model.py diff --git a/sana/sana_1600M/packages/Sana/CITATION.bib b/image/sana/sana-1600m/packages/Sana/CITATION.bib similarity index 100% rename from sana/sana_1600M/packages/Sana/CITATION.bib rename to image/sana/sana-1600m/packages/Sana/CITATION.bib diff --git a/sana/sana_1600M/packages/Sana/CIs/add_license_all.sh b/image/sana/sana-1600m/packages/Sana/CIs/add_license_all.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/CIs/add_license_all.sh rename to image/sana/sana-1600m/packages/Sana/CIs/add_license_all.sh diff --git a/sana/sana_1600M/packages/Sana/Dockerfile b/image/sana/sana-1600m/packages/Sana/Dockerfile similarity index 100% rename from sana/sana_1600M/packages/Sana/Dockerfile rename to image/sana/sana-1600m/packages/Sana/Dockerfile diff --git a/sana/sana_1600M/packages/Sana/LICENSE b/image/sana/sana-1600m/packages/Sana/LICENSE similarity index 100% rename from sana/sana_1600M/packages/Sana/LICENSE rename to image/sana/sana-1600m/packages/Sana/LICENSE diff --git a/sana/sana_1600M/packages/Sana/README.md b/image/sana/sana-1600m/packages/Sana/README.md similarity index 100% rename from sana/sana_1600M/packages/Sana/README.md rename to image/sana/sana-1600m/packages/Sana/README.md diff --git a/sana/sana_1600M/packages/Sana/app/app_sana.py b/image/sana/sana-1600m/packages/Sana/app/app_sana.py similarity index 100% rename from sana/sana_1600M/packages/Sana/app/app_sana.py rename to image/sana/sana-1600m/packages/Sana/app/app_sana.py diff --git a/sana/sana_1600M/packages/Sana/app/app_sana_multithread.py b/image/sana/sana-1600m/packages/Sana/app/app_sana_multithread.py similarity index 100% rename from sana/sana_1600M/packages/Sana/app/app_sana_multithread.py rename to image/sana/sana-1600m/packages/Sana/app/app_sana_multithread.py diff --git a/sana/sana_1600M/packages/Sana/app/safety_check.py b/image/sana/sana-1600m/packages/Sana/app/safety_check.py similarity index 100% rename from sana/sana_1600M/packages/Sana/app/safety_check.py rename to image/sana/sana-1600m/packages/Sana/app/safety_check.py diff --git a/sana/sana_1600M/packages/Sana/app/sana_pipeline.py b/image/sana/sana-1600m/packages/Sana/app/sana_pipeline.py similarity index 100% rename from sana/sana_1600M/packages/Sana/app/sana_pipeline.py rename to image/sana/sana-1600m/packages/Sana/app/sana_pipeline.py diff --git a/sana/sana_1600M/packages/Sana/asset/Sana.jpg b/image/sana/sana-1600m/packages/Sana/asset/Sana.jpg similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/Sana.jpg rename to image/sana/sana-1600m/packages/Sana/asset/Sana.jpg diff --git a/sana/sana_1600M/packages/Sana/asset/docs/metrics_toolkit.md b/image/sana/sana-1600m/packages/Sana/asset/docs/metrics_toolkit.md similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/docs/metrics_toolkit.md rename to image/sana/sana-1600m/packages/Sana/asset/docs/metrics_toolkit.md diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/00000000.png b/image/sana/sana-1600m/packages/Sana/asset/example_data/00000000.png similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/00000000.png rename to image/sana/sana-1600m/packages/Sana/asset/example_data/00000000.png diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/00000000.txt b/image/sana/sana-1600m/packages/Sana/asset/example_data/00000000.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/00000000.txt rename to image/sana/sana-1600m/packages/Sana/asset/example_data/00000000.txt diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/00000000_InternVL2-26B.json b/image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_InternVL2-26B.json similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/00000000_InternVL2-26B.json rename to image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_InternVL2-26B.json diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json b/image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json rename to image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json b/image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json rename to image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json b/image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json rename to image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/00000000_prompt_clip_score.json b/image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_prompt_clip_score.json similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/00000000_prompt_clip_score.json rename to image/sana/sana-1600m/packages/Sana/asset/example_data/00000000_prompt_clip_score.json diff --git a/sana/sana_1600M/packages/Sana/asset/example_data/meta_data.json b/image/sana/sana-1600m/packages/Sana/asset/example_data/meta_data.json similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/example_data/meta_data.json rename to image/sana/sana-1600m/packages/Sana/asset/example_data/meta_data.json diff --git a/sana/sana_1600M/packages/Sana/asset/examples.py b/image/sana/sana-1600m/packages/Sana/asset/examples.py similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/examples.py rename to image/sana/sana-1600m/packages/Sana/asset/examples.py diff --git a/sana/sana_1600M/packages/Sana/asset/logo.png b/image/sana/sana-1600m/packages/Sana/asset/logo.png similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/logo.png rename to image/sana/sana-1600m/packages/Sana/asset/logo.png diff --git a/sana/sana_1600M/packages/Sana/asset/model-incremental.jpg b/image/sana/sana-1600m/packages/Sana/asset/model-incremental.jpg similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/model-incremental.jpg rename to image/sana/sana-1600m/packages/Sana/asset/model-incremental.jpg diff --git a/sana/sana_1600M/packages/Sana/asset/model_paths.txt b/image/sana/sana-1600m/packages/Sana/asset/model_paths.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/model_paths.txt rename to image/sana/sana-1600m/packages/Sana/asset/model_paths.txt diff --git a/sana/sana_1600M/packages/Sana/asset/samples.txt b/image/sana/sana-1600m/packages/Sana/asset/samples.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/samples.txt rename to image/sana/sana-1600m/packages/Sana/asset/samples.txt diff --git a/sana/sana_1600M/packages/Sana/asset/samples_mini.txt b/image/sana/sana-1600m/packages/Sana/asset/samples_mini.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/asset/samples_mini.txt rename to image/sana/sana-1600m/packages/Sana/asset/samples_mini.txt diff --git a/sana/sana_1600M/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_base.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_base.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_base.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_base.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml diff --git a/sana/sana_1600M/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml b/image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml similarity index 100% rename from sana/sana_1600M/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml rename to image/sana/sana-1600m/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml diff --git a/sana/sana_1600M/packages/Sana/diffusion/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/builder.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/builder.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/builder.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/builder.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/datasets/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/datasets/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/datasets/sana_data.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/sana_data.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/datasets/sana_data.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/sana_data.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/datasets/utils.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/utils.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/datasets/utils.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/datasets/utils.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/transforms.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/transforms.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/transforms.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/transforms.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/wids/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/wids/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/wids/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/wids/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/wids/wids.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/wids/wids.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_dl.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_dl.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_dl.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_dl.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_lru.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_lru.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_lru.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_lru.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_mmtar.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_mmtar.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_mmtar.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_mmtar.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_specs.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_specs.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_specs.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_specs.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_tar.py b/image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_tar.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/data/wids/wids_tar.py rename to image/sana/sana-1600m/packages/Sana/diffusion/data/wids/wids_tar.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/dpm_solver.py b/image/sana/sana-1600m/packages/Sana/diffusion/dpm_solver.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/dpm_solver.py rename to image/sana/sana-1600m/packages/Sana/diffusion/dpm_solver.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/flow_euler_sampler.py b/image/sana/sana-1600m/packages/Sana/diffusion/flow_euler_sampler.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/flow_euler_sampler.py rename to image/sana/sana-1600m/packages/Sana/diffusion/flow_euler_sampler.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/iddpm.py b/image/sana/sana-1600m/packages/Sana/diffusion/iddpm.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/iddpm.py rename to image/sana/sana-1600m/packages/Sana/diffusion/iddpm.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/lcm_scheduler.py b/image/sana/sana-1600m/packages/Sana/diffusion/lcm_scheduler.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/lcm_scheduler.py rename to image/sana/sana-1600m/packages/Sana/diffusion/lcm_scheduler.py diff --git a/llama/llama-2-70b/model/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from llama/llama-2-70b/model/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/act.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/act.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/act.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/act.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/builder.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/builder.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/builder.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/builder.py diff --git a/llama/llama-2-7b-chat/model/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py similarity index 100% rename from llama/llama-2-7b-chat/model/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py diff --git a/llama/llama-2-7b/model/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py similarity index 100% rename from llama/llama-2-7b/model/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py diff --git a/llama/llama-3-70b-instruct/model/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py similarity index 100% rename from llama/llama-3-70b-instruct/model/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/diffusion_utils.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/diffusion_utils.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/diffusion_utils.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/diffusion_utils.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dpm_solver.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dpm_solver.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dpm_solver.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/dpm_solver.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/edm_sample.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/edm_sample.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/edm_sample.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/edm_sample.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/gaussian_diffusion.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/gaussian_diffusion.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/gaussian_diffusion.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/gaussian_diffusion.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/basic_modules.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/basic_modules.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/basic_modules.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/basic_modules.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/readme.md b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/readme.md similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/fastlinear/readme.md rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/fastlinear/readme.md diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/sana.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/sana.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_U_shape.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_U_shape.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_U_shape.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_U_shape.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_blocks.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_blocks.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_blocks.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_blocks.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_multi_scale.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_multi_scale.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_multi_scale.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_multi_scale.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_others.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_others.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/nets/sana_others.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/nets/sana_others.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/norms.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/norms.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/norms.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/norms.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/respace.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/respace.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/respace.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/respace.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/sa_solver.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/sa_solver.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/sa_solver.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/sa_solver.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/timestep_sampler.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/timestep_sampler.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/timestep_sampler.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/timestep_sampler.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/utils.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/utils.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/utils.py rename to image/sana/sana-1600m/packages/Sana/diffusion/model/utils.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/sa_sampler.py b/image/sana/sana-1600m/packages/Sana/diffusion/sa_sampler.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/sa_sampler.py rename to image/sana/sana-1600m/packages/Sana/diffusion/sa_sampler.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/sa_solver_diffusers.py b/image/sana/sana-1600m/packages/Sana/diffusion/sa_solver_diffusers.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/sa_solver_diffusers.py rename to image/sana/sana-1600m/packages/Sana/diffusion/sa_solver_diffusers.py diff --git a/llama/llama-3-8b-instruct/model/__init__.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from llama/llama-3-8b-instruct/model/__init__.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/__init__.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/checkpoint.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/checkpoint.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/checkpoint.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/checkpoint.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/config.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/config.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/config.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/config.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/data_sampler.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/data_sampler.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/data_sampler.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/data_sampler.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/dist_utils.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/dist_utils.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/dist_utils.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/dist_utils.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/import_utils.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/import_utils.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/import_utils.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/import_utils.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/logger.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/logger.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/logger.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/logger.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/lr_scheduler.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/lr_scheduler.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/lr_scheduler.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/lr_scheduler.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/misc.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/misc.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/misc.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/misc.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/optimizer.py b/image/sana/sana-1600m/packages/Sana/diffusion/utils/optimizer.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/optimizer.py rename to image/sana/sana-1600m/packages/Sana/diffusion/utils/optimizer.py diff --git a/sana/sana_1600M/packages/Sana/environment_setup.sh b/image/sana/sana-1600m/packages/Sana/environment_setup.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/environment_setup.sh rename to image/sana/sana-1600m/packages/Sana/environment_setup.sh diff --git a/sana/sana_1600M/packages/Sana/pyproject.toml b/image/sana/sana-1600m/packages/Sana/pyproject.toml similarity index 100% rename from sana/sana_1600M/packages/Sana/pyproject.toml rename to image/sana/sana-1600m/packages/Sana/pyproject.toml diff --git a/sana/sana_1600M/packages/Sana/sana/cli/run.py b/image/sana/sana-1600m/packages/Sana/sana/cli/run.py similarity index 100% rename from sana/sana_1600M/packages/Sana/sana/cli/run.py rename to image/sana/sana-1600m/packages/Sana/sana/cli/run.py diff --git a/sana/sana_1600M/packages/Sana/sana/cli/upload2hf.py b/image/sana/sana-1600m/packages/Sana/sana/cli/upload2hf.py similarity index 100% rename from sana/sana_1600M/packages/Sana/sana/cli/upload2hf.py rename to image/sana/sana-1600m/packages/Sana/sana/cli/upload2hf.py diff --git a/sana/sana_1600M/packages/Sana/sana/tools/__init__.py b/image/sana/sana-1600m/packages/Sana/sana/tools/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/sana/tools/__init__.py rename to image/sana/sana-1600m/packages/Sana/sana/tools/__init__.py diff --git a/sana/sana_1600M/packages/Sana/sana/tools/download.py b/image/sana/sana-1600m/packages/Sana/sana/tools/download.py similarity index 100% rename from sana/sana_1600M/packages/Sana/sana/tools/download.py rename to image/sana/sana-1600m/packages/Sana/sana/tools/download.py diff --git a/sana/sana_1600M/packages/Sana/sana/tools/hf_utils.py b/image/sana/sana-1600m/packages/Sana/sana/tools/hf_utils.py similarity index 100% rename from sana/sana_1600M/packages/Sana/sana/tools/hf_utils.py rename to image/sana/sana-1600m/packages/Sana/sana/tools/hf_utils.py diff --git a/sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric.sh b/image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric.sh rename to image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric_dpg.sh b/image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric_dpg.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric_dpg.sh rename to image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric_dpg.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric_geneval.sh b/image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric_geneval.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric_geneval.sh rename to image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric_geneval.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh b/image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh rename to image/sana/sana-1600m/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/infer_metric_run_inference_metric.sh b/image/sana/sana-1600m/packages/Sana/scripts/infer_metric_run_inference_metric.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/infer_metric_run_inference_metric.sh rename to image/sana/sana-1600m/packages/Sana/scripts/infer_metric_run_inference_metric.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh b/image/sana/sana-1600m/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh rename to image/sana/sana-1600m/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/infer_run_inference.sh b/image/sana/sana-1600m/packages/Sana/scripts/infer_run_inference.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/infer_run_inference.sh rename to image/sana/sana-1600m/packages/Sana/scripts/infer_run_inference.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/infer_run_inference_geneval.sh b/image/sana/sana-1600m/packages/Sana/scripts/infer_run_inference_geneval.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/infer_run_inference_geneval.sh rename to image/sana/sana-1600m/packages/Sana/scripts/infer_run_inference_geneval.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh b/image/sana/sana-1600m/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh rename to image/sana/sana-1600m/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh diff --git a/sana/sana_1600M/packages/Sana/scripts/inference.py b/image/sana/sana-1600m/packages/Sana/scripts/inference.py similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/inference.py rename to image/sana/sana-1600m/packages/Sana/scripts/inference.py diff --git a/sana/sana_1600M/packages/Sana/scripts/inference_dpg.py b/image/sana/sana-1600m/packages/Sana/scripts/inference_dpg.py similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/inference_dpg.py rename to image/sana/sana-1600m/packages/Sana/scripts/inference_dpg.py diff --git a/sana/sana_1600M/packages/Sana/scripts/inference_geneval.py b/image/sana/sana-1600m/packages/Sana/scripts/inference_geneval.py similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/inference_geneval.py rename to image/sana/sana-1600m/packages/Sana/scripts/inference_geneval.py diff --git a/sana/sana_1600M/packages/Sana/scripts/inference_geneval_diffusers.py b/image/sana/sana-1600m/packages/Sana/scripts/inference_geneval_diffusers.py similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/inference_geneval_diffusers.py rename to image/sana/sana-1600m/packages/Sana/scripts/inference_geneval_diffusers.py diff --git a/sana/sana_1600M/packages/Sana/scripts/inference_image_reward.py b/image/sana/sana-1600m/packages/Sana/scripts/inference_image_reward.py similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/inference_image_reward.py rename to image/sana/sana-1600m/packages/Sana/scripts/inference_image_reward.py diff --git a/sana/sana_1600M/packages/Sana/scripts/interface.py b/image/sana/sana-1600m/packages/Sana/scripts/interface.py similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/interface.py rename to image/sana/sana-1600m/packages/Sana/scripts/interface.py diff --git a/sana/sana_1600M/packages/Sana/scripts/style.css b/image/sana/sana-1600m/packages/Sana/scripts/style.css similarity index 100% rename from sana/sana_1600M/packages/Sana/scripts/style.css rename to image/sana/sana-1600m/packages/Sana/scripts/style.css diff --git a/sana/sana_1600M/packages/Sana/tests/bash/entry.sh b/image/sana/sana-1600m/packages/Sana/tests/bash/entry.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tests/bash/entry.sh rename to image/sana/sana-1600m/packages/Sana/tests/bash/entry.sh diff --git a/sana/sana_1600M/packages/Sana/tests/bash/test_inference.sh b/image/sana/sana-1600m/packages/Sana/tests/bash/test_inference.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tests/bash/test_inference.sh rename to image/sana/sana-1600m/packages/Sana/tests/bash/test_inference.sh diff --git a/sana/sana_1600M/packages/Sana/tests/bash/test_training_1epoch.sh b/image/sana/sana-1600m/packages/Sana/tests/bash/test_training_1epoch.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tests/bash/test_training_1epoch.sh rename to image/sana/sana-1600m/packages/Sana/tests/bash/test_training_1epoch.sh diff --git a/llama/llama-3_1-405b-instruct/model/__init__.py b/image/sana/sana-1600m/packages/Sana/tools/__init__.py similarity index 100% rename from llama/llama-3_1-405b-instruct/model/__init__.py rename to image/sana/sana-1600m/packages/Sana/tools/__init__.py diff --git a/sana/sana_1600M/packages/Sana/tools/convert_py_to_yaml.py b/image/sana/sana-1600m/packages/Sana/tools/convert_py_to_yaml.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/convert_py_to_yaml.py rename to image/sana/sana-1600m/packages/Sana/tools/convert_py_to_yaml.py diff --git a/sana/sana_1600M/packages/Sana/tools/convert_sana_pag_to_diffusers.py b/image/sana/sana-1600m/packages/Sana/tools/convert_sana_pag_to_diffusers.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/convert_sana_pag_to_diffusers.py rename to image/sana/sana-1600m/packages/Sana/tools/convert_sana_pag_to_diffusers.py diff --git a/sana/sana_1600M/packages/Sana/tools/convert_sana_to_diffusers.py b/image/sana/sana-1600m/packages/Sana/tools/convert_sana_to_diffusers.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/convert_sana_to_diffusers.py rename to image/sana/sana-1600m/packages/Sana/tools/convert_sana_to_diffusers.py diff --git a/sana/sana_1600M/packages/Sana/tools/download.py b/image/sana/sana-1600m/packages/Sana/tools/download.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/download.py rename to image/sana/sana-1600m/packages/Sana/tools/download.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/.gitignore b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/.gitignore similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/.gitignore rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/.gitignore diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/LICENSE b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/LICENSE similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/LICENSE rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/LICENSE diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/README.md b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/README.md similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/README.md rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/README.md diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/clip_score.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/clip_score.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/clip_score.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/clip_score.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/setup.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/setup.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/setup.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/setup.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/compute_clipscore.sh b/image/sana/sana-1600m/packages/Sana/tools/metrics/compute_clipscore.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/compute_clipscore.sh rename to image/sana/sana-1600m/packages/Sana/tools/metrics/compute_clipscore.sh diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/compute_dpg.sh b/image/sana/sana-1600m/packages/Sana/tools/metrics/compute_dpg.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/compute_dpg.sh rename to image/sana/sana-1600m/packages/Sana/tools/metrics/compute_dpg.sh diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/compute_fid_embedding.sh b/image/sana/sana-1600m/packages/Sana/tools/metrics/compute_fid_embedding.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/compute_fid_embedding.sh rename to image/sana/sana-1600m/packages/Sana/tools/metrics/compute_fid_embedding.sh diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/compute_geneval.sh b/image/sana/sana-1600m/packages/Sana/tools/metrics/compute_geneval.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/compute_geneval.sh rename to image/sana/sana-1600m/packages/Sana/tools/metrics/compute_geneval.sh diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/compute_imagereward.sh b/image/sana/sana-1600m/packages/Sana/tools/metrics/compute_imagereward.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/compute_imagereward.sh rename to image/sana/sana-1600m/packages/Sana/tools/metrics/compute_imagereward.sh diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv b/image/sana/sana-1600m/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv rename to image/sana/sana-1600m/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/dpg_bench/requirements.txt b/image/sana/sana-1600m/packages/Sana/tools/metrics/dpg_bench/requirements.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/dpg_bench/requirements.txt rename to image/sana/sana-1600m/packages/Sana/tools/metrics/dpg_bench/requirements.txt diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/LICENSE b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/LICENSE similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/LICENSE rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/LICENSE diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/README.md b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/README.md similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/README.md rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/README.md diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/environment.yml b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/environment.yml similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/environment.yml rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/environment.yml diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/object_names.txt b/image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/object_names.txt similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/geneval/prompts/object_names.txt rename to image/sana/sana-1600m/packages/Sana/tools/metrics/geneval/prompts/object_names.txt diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json b/image/sana/sana-1600m/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json rename to image/sana/sana-1600m/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/image_reward/compute_image_reward.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/image_reward/compute_image_reward.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/image_reward/compute_image_reward.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/image_reward/compute_image_reward.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/.gitignore b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/.gitignore similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/.gitignore rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/.gitignore diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/LICENSE b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/LICENSE similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/LICENSE rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/LICENSE diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/README.md b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/README.md similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/README.md rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/README.md diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/noxfile.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/noxfile.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/noxfile.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/noxfile.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/setup.cfg b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/setup.cfg similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/setup.cfg rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/setup.cfg diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/setup.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/setup.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/setup.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/setup.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py diff --git a/sana/sana_1600M/packages/Sana/tools/metrics/utils.py b/image/sana/sana-1600m/packages/Sana/tools/metrics/utils.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/metrics/utils.py rename to image/sana/sana-1600m/packages/Sana/tools/metrics/utils.py diff --git a/sana/sana_1600M/packages/Sana/train_scripts/train.py b/image/sana/sana-1600m/packages/Sana/train_scripts/train.py similarity index 100% rename from sana/sana_1600M/packages/Sana/train_scripts/train.py rename to image/sana/sana-1600m/packages/Sana/train_scripts/train.py diff --git a/sana/sana_1600M/packages/Sana/train_scripts/train.sh b/image/sana/sana-1600m/packages/Sana/train_scripts/train.sh similarity index 100% rename from sana/sana_1600M/packages/Sana/train_scripts/train.sh rename to image/sana/sana-1600m/packages/Sana/train_scripts/train.sh diff --git a/image/sana/sana-600m/README.md b/image/sana/sana-600m/README.md new file mode 100644 index 000000000..2f83d6f71 --- /dev/null +++ b/image/sana/sana-600m/README.md @@ -0,0 +1,40 @@ +# Sana 600M + +Deploy Sana 600M for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | H100_40GB | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "a photo of an astronaut riding a horse on mars", + "height": 1024, + "width": 1024, + "guidance_scale": 5.0, + "pag_guidance_scale": 2.0, + "num_inference_steps": 18, + "seed": 4096 +}' +``` + +## Configuration highlights + +- Base image: `alphatozeta/cuda-python:12.1.1-cudnn8-devel-ubuntu22.04` +- System packages: `ffmpeg, libsm6, libxext6, python3.10-venv` diff --git a/sana/sana_600M/config.yaml b/image/sana/sana-600m/config.yaml similarity index 87% rename from sana/sana_600M/config.yaml rename to image/sana/sana-600m/config.yaml index fb9afda13..eb0174851 100644 --- a/sana/sana_600M/config.yaml +++ b/image/sana/sana-600m/config.yaml @@ -1,9 +1,11 @@ +description: "Sana 600M for image generation" build_commands: [] base_image: image: alphatozeta/cuda-python:12.1.1-cudnn8-devel-ubuntu22.04 environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "Efficient-Large-Model/Sana_600M_1024px_diffusers" example_model_input: { "prompt": "a photo of an astronaut riding a horse on mars", "height": 1024, diff --git a/llama/llama-3_1-8b-instruct-sglang/model/__init__.py b/image/sana/sana-600m/model/__init__.py similarity index 100% rename from llama/llama-3_1-8b-instruct-sglang/model/__init__.py rename to image/sana/sana-600m/model/__init__.py diff --git a/sana/sana_600M/model/model.py b/image/sana/sana-600m/model/model.py similarity index 100% rename from sana/sana_600M/model/model.py rename to image/sana/sana-600m/model/model.py diff --git a/sana/sana_600M/packages/Sana/CITATION.bib b/image/sana/sana-600m/packages/Sana/CITATION.bib similarity index 100% rename from sana/sana_600M/packages/Sana/CITATION.bib rename to image/sana/sana-600m/packages/Sana/CITATION.bib diff --git a/sana/sana_600M/packages/Sana/CIs/add_license_all.sh b/image/sana/sana-600m/packages/Sana/CIs/add_license_all.sh similarity index 100% rename from sana/sana_600M/packages/Sana/CIs/add_license_all.sh rename to image/sana/sana-600m/packages/Sana/CIs/add_license_all.sh diff --git a/sana/sana_600M/packages/Sana/Dockerfile b/image/sana/sana-600m/packages/Sana/Dockerfile similarity index 100% rename from sana/sana_600M/packages/Sana/Dockerfile rename to image/sana/sana-600m/packages/Sana/Dockerfile diff --git a/sana/sana_600M/packages/Sana/LICENSE b/image/sana/sana-600m/packages/Sana/LICENSE similarity index 100% rename from sana/sana_600M/packages/Sana/LICENSE rename to image/sana/sana-600m/packages/Sana/LICENSE diff --git a/sana/sana_600M/packages/Sana/README.md b/image/sana/sana-600m/packages/Sana/README.md similarity index 100% rename from sana/sana_600M/packages/Sana/README.md rename to image/sana/sana-600m/packages/Sana/README.md diff --git a/sana/sana_600M/packages/Sana/app/app_sana.py b/image/sana/sana-600m/packages/Sana/app/app_sana.py similarity index 100% rename from sana/sana_600M/packages/Sana/app/app_sana.py rename to image/sana/sana-600m/packages/Sana/app/app_sana.py diff --git a/sana/sana_600M/packages/Sana/app/app_sana_multithread.py b/image/sana/sana-600m/packages/Sana/app/app_sana_multithread.py similarity index 100% rename from sana/sana_600M/packages/Sana/app/app_sana_multithread.py rename to image/sana/sana-600m/packages/Sana/app/app_sana_multithread.py diff --git a/sana/sana_600M/packages/Sana/app/safety_check.py b/image/sana/sana-600m/packages/Sana/app/safety_check.py similarity index 100% rename from sana/sana_600M/packages/Sana/app/safety_check.py rename to image/sana/sana-600m/packages/Sana/app/safety_check.py diff --git a/sana/sana_600M/packages/Sana/app/sana_pipeline.py b/image/sana/sana-600m/packages/Sana/app/sana_pipeline.py similarity index 100% rename from sana/sana_600M/packages/Sana/app/sana_pipeline.py rename to image/sana/sana-600m/packages/Sana/app/sana_pipeline.py diff --git a/sana/sana_600M/packages/Sana/asset/Sana.jpg b/image/sana/sana-600m/packages/Sana/asset/Sana.jpg similarity index 100% rename from sana/sana_600M/packages/Sana/asset/Sana.jpg rename to image/sana/sana-600m/packages/Sana/asset/Sana.jpg diff --git a/sana/sana_600M/packages/Sana/asset/docs/metrics_toolkit.md b/image/sana/sana-600m/packages/Sana/asset/docs/metrics_toolkit.md similarity index 100% rename from sana/sana_600M/packages/Sana/asset/docs/metrics_toolkit.md rename to image/sana/sana-600m/packages/Sana/asset/docs/metrics_toolkit.md diff --git a/sana/sana_600M/packages/Sana/asset/example_data/00000000.png b/image/sana/sana-600m/packages/Sana/asset/example_data/00000000.png similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/00000000.png rename to image/sana/sana-600m/packages/Sana/asset/example_data/00000000.png diff --git a/sana/sana_600M/packages/Sana/asset/example_data/00000000.txt b/image/sana/sana-600m/packages/Sana/asset/example_data/00000000.txt similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/00000000.txt rename to image/sana/sana-600m/packages/Sana/asset/example_data/00000000.txt diff --git a/sana/sana_600M/packages/Sana/asset/example_data/00000000_InternVL2-26B.json b/image/sana/sana-600m/packages/Sana/asset/example_data/00000000_InternVL2-26B.json similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/00000000_InternVL2-26B.json rename to image/sana/sana-600m/packages/Sana/asset/example_data/00000000_InternVL2-26B.json diff --git a/sana/sana_600M/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json b/image/sana/sana-600m/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json rename to image/sana/sana-600m/packages/Sana/asset/example_data/00000000_InternVL2-26B_clip_score.json diff --git a/sana/sana_600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json b/image/sana/sana-600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json rename to image/sana/sana-600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B.json diff --git a/sana/sana_600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json b/image/sana/sana-600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json rename to image/sana/sana-600m/packages/Sana/asset/example_data/00000000_VILA1-5-13B_clip_score.json diff --git a/sana/sana_600M/packages/Sana/asset/example_data/00000000_prompt_clip_score.json b/image/sana/sana-600m/packages/Sana/asset/example_data/00000000_prompt_clip_score.json similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/00000000_prompt_clip_score.json rename to image/sana/sana-600m/packages/Sana/asset/example_data/00000000_prompt_clip_score.json diff --git a/sana/sana_600M/packages/Sana/asset/example_data/meta_data.json b/image/sana/sana-600m/packages/Sana/asset/example_data/meta_data.json similarity index 100% rename from sana/sana_600M/packages/Sana/asset/example_data/meta_data.json rename to image/sana/sana-600m/packages/Sana/asset/example_data/meta_data.json diff --git a/sana/sana_600M/packages/Sana/asset/examples.py b/image/sana/sana-600m/packages/Sana/asset/examples.py similarity index 100% rename from sana/sana_600M/packages/Sana/asset/examples.py rename to image/sana/sana-600m/packages/Sana/asset/examples.py diff --git a/sana/sana_600M/packages/Sana/asset/logo.png b/image/sana/sana-600m/packages/Sana/asset/logo.png similarity index 100% rename from sana/sana_600M/packages/Sana/asset/logo.png rename to image/sana/sana-600m/packages/Sana/asset/logo.png diff --git a/sana/sana_600M/packages/Sana/asset/model-incremental.jpg b/image/sana/sana-600m/packages/Sana/asset/model-incremental.jpg similarity index 100% rename from sana/sana_600M/packages/Sana/asset/model-incremental.jpg rename to image/sana/sana-600m/packages/Sana/asset/model-incremental.jpg diff --git a/sana/sana_600M/packages/Sana/asset/model_paths.txt b/image/sana/sana-600m/packages/Sana/asset/model_paths.txt similarity index 100% rename from sana/sana_600M/packages/Sana/asset/model_paths.txt rename to image/sana/sana-600m/packages/Sana/asset/model_paths.txt diff --git a/sana/sana_600M/packages/Sana/asset/samples.txt b/image/sana/sana-600m/packages/Sana/asset/samples.txt similarity index 100% rename from sana/sana_600M/packages/Sana/asset/samples.txt rename to image/sana/sana-600m/packages/Sana/asset/samples.txt diff --git a/sana/sana_600M/packages/Sana/asset/samples_mini.txt b/image/sana/sana-600m/packages/Sana/asset/samples_mini.txt similarity index 100% rename from sana/sana_600M/packages/Sana/asset/samples_mini.txt rename to image/sana/sana-600m/packages/Sana/asset/samples_mini.txt diff --git a/sana/sana_600M/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_app_config/Sana_1600M_app.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_app_config/Sana_600M_app.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_base.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_base.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_base.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_base.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_config/1024ms/Sana_1600M_img1024_AdamW.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_config/1024ms/Sana_600M_img1024.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/Sana_1600M_img512.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/Sana_600M_img512.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/ci_Sana_600M_img512.yaml diff --git a/sana/sana_600M/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml b/image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml similarity index 100% rename from sana/sana_600M/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml rename to image/sana/sana-600m/packages/Sana/configs/sana_config/512ms/sample_dataset.yaml diff --git a/sana/sana_600M/packages/Sana/diffusion/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/data/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/builder.py b/image/sana/sana-600m/packages/Sana/diffusion/data/builder.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/builder.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/builder.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/datasets/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/data/datasets/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/datasets/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/datasets/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/datasets/sana_data.py b/image/sana/sana-600m/packages/Sana/diffusion/data/datasets/sana_data.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/datasets/sana_data.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/datasets/sana_data.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py b/image/sana/sana-600m/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/datasets/sana_data_multi_scale.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/datasets/utils.py b/image/sana/sana-600m/packages/Sana/diffusion/data/datasets/utils.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/datasets/utils.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/datasets/utils.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/transforms.py b/image/sana/sana-600m/packages/Sana/diffusion/data/transforms.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/transforms.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/transforms.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/wids/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/data/wids/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/wids/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/wids/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/wids/wids.py b/image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/wids/wids.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/wids/wids_dl.py b/image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_dl.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/wids/wids_dl.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_dl.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/wids/wids_lru.py b/image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_lru.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/wids/wids_lru.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_lru.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/wids/wids_mmtar.py b/image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_mmtar.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/wids/wids_mmtar.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_mmtar.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/wids/wids_specs.py b/image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_specs.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/wids/wids_specs.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_specs.py diff --git a/sana/sana_600M/packages/Sana/diffusion/data/wids/wids_tar.py b/image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_tar.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/data/wids/wids_tar.py rename to image/sana/sana-600m/packages/Sana/diffusion/data/wids/wids_tar.py diff --git a/sana/sana_600M/packages/Sana/diffusion/dpm_solver.py b/image/sana/sana-600m/packages/Sana/diffusion/dpm_solver.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/dpm_solver.py rename to image/sana/sana-600m/packages/Sana/diffusion/dpm_solver.py diff --git a/sana/sana_600M/packages/Sana/diffusion/flow_euler_sampler.py b/image/sana/sana-600m/packages/Sana/diffusion/flow_euler_sampler.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/flow_euler_sampler.py rename to image/sana/sana-600m/packages/Sana/diffusion/flow_euler_sampler.py diff --git a/sana/sana_600M/packages/Sana/diffusion/iddpm.py b/image/sana/sana-600m/packages/Sana/diffusion/iddpm.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/iddpm.py rename to image/sana/sana-600m/packages/Sana/diffusion/iddpm.py diff --git a/sana/sana_600M/packages/Sana/diffusion/lcm_scheduler.py b/image/sana/sana-600m/packages/Sana/diffusion/lcm_scheduler.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/lcm_scheduler.py rename to image/sana/sana-600m/packages/Sana/diffusion/lcm_scheduler.py diff --git a/llama/llama-3_1-8b-instruct/model/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from llama/llama-3_1-8b-instruct/model/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/act.py b/image/sana/sana-600m/packages/Sana/diffusion/model/act.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/act.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/act.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/builder.py b/image/sana/sana-600m/packages/Sana/diffusion/model/builder.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/builder.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/builder.py diff --git a/llama/llama-3_1_70b-instruct/model/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py similarity index 100% rename from llama/llama-3_1_70b-instruct/model/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/ae_model_zoo.py diff --git a/llama/llama-7b-exllama-streaming/model/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py similarity index 100% rename from llama/llama-7b-exllama-streaming/model/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/setup.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/trainer/run_config.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/dist.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/ema.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/export.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/image.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/init.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/lr.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/metric.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/misc.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/utils/opt.py diff --git a/llama/llama-7b-exllama/model/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py similarity index 100% rename from llama/llama-7b-exllama/model/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/efficientvit/dc_ae.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/act.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/drop.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/norm.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/ops.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/nn/triton_rms_norm.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/list.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/network.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dc_ae/efficientvit/models/utils/random.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/diffusion_utils.py b/image/sana/sana-600m/packages/Sana/diffusion/model/diffusion_utils.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/diffusion_utils.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/diffusion_utils.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dpm_solver.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dpm_solver.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dpm_solver.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/dpm_solver.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/edm_sample.py b/image/sana/sana-600m/packages/Sana/diffusion/model/edm_sample.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/edm_sample.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/edm_sample.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/gaussian_diffusion.py b/image/sana/sana-600m/packages/Sana/diffusion/model/gaussian_diffusion.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/gaussian_diffusion.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/gaussian_diffusion.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/basic_modules.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/basic_modules.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/basic_modules.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/basic_modules.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_ffn.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/develop_triton_litemla.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/flash_attn.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/lite_mla.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/mb_conv_pre_glu.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/act.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/conv.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/nn/norm.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_fwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/custom_autotune.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/linear_relu_fwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/mm.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/pad_vk_mm_fwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/proj_divide_bwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_mm_relu_bwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_divide_fwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_lite_mla_kernels/vk_q_mm_relu_bwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/depthwise_conv_fwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/triton_mb_conv_pre_glu_kernels/linear_glu_fwd.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/compare_results.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/custom_autotune.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/dtype.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/export_onnx.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/modules/utils/model.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/readme.md b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/readme.md similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/fastlinear/readme.md rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/fastlinear/readme.md diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/sana.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/sana.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/sana_U_shape.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_U_shape.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/sana_U_shape.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_U_shape.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_U_shape_multi_scale.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/sana_blocks.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_blocks.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/sana_blocks.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_blocks.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/sana_multi_scale.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_multi_scale.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/sana_multi_scale.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_multi_scale.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_multi_scale_adaln.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/nets/sana_others.py b/image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_others.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/nets/sana_others.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/nets/sana_others.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/norms.py b/image/sana/sana-600m/packages/Sana/diffusion/model/norms.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/norms.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/norms.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/respace.py b/image/sana/sana-600m/packages/Sana/diffusion/model/respace.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/respace.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/respace.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/sa_solver.py b/image/sana/sana-600m/packages/Sana/diffusion/model/sa_solver.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/sa_solver.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/sa_solver.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/timestep_sampler.py b/image/sana/sana-600m/packages/Sana/diffusion/model/timestep_sampler.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/timestep_sampler.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/timestep_sampler.py diff --git a/sana/sana_600M/packages/Sana/diffusion/model/utils.py b/image/sana/sana-600m/packages/Sana/diffusion/model/utils.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/utils.py rename to image/sana/sana-600m/packages/Sana/diffusion/model/utils.py diff --git a/sana/sana_600M/packages/Sana/diffusion/sa_sampler.py b/image/sana/sana-600m/packages/Sana/diffusion/sa_sampler.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/sa_sampler.py rename to image/sana/sana-600m/packages/Sana/diffusion/sa_sampler.py diff --git a/sana/sana_600M/packages/Sana/diffusion/sa_solver_diffusers.py b/image/sana/sana-600m/packages/Sana/diffusion/sa_solver_diffusers.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/sa_solver_diffusers.py rename to image/sana/sana-600m/packages/Sana/diffusion/sa_solver_diffusers.py diff --git a/llama/llama-7b-vllm/model/__init__.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from llama/llama-7b-vllm/model/__init__.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/__init__.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/checkpoint.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/checkpoint.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/checkpoint.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/checkpoint.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/config.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/config.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/config.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/config.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/data_sampler.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/data_sampler.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/data_sampler.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/data_sampler.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/dist_utils.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/dist_utils.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/dist_utils.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/dist_utils.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/import_utils.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/import_utils.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/import_utils.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/import_utils.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/logger.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/logger.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/logger.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/logger.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/lr_scheduler.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/lr_scheduler.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/lr_scheduler.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/lr_scheduler.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/misc.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/misc.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/misc.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/misc.py diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/optimizer.py b/image/sana/sana-600m/packages/Sana/diffusion/utils/optimizer.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/optimizer.py rename to image/sana/sana-600m/packages/Sana/diffusion/utils/optimizer.py diff --git a/sana/sana_600M/packages/Sana/environment_setup.sh b/image/sana/sana-600m/packages/Sana/environment_setup.sh similarity index 100% rename from sana/sana_600M/packages/Sana/environment_setup.sh rename to image/sana/sana-600m/packages/Sana/environment_setup.sh diff --git a/sana/sana_600M/packages/Sana/pyproject.toml b/image/sana/sana-600m/packages/Sana/pyproject.toml similarity index 100% rename from sana/sana_600M/packages/Sana/pyproject.toml rename to image/sana/sana-600m/packages/Sana/pyproject.toml diff --git a/sana/sana_600M/packages/Sana/sana/cli/run.py b/image/sana/sana-600m/packages/Sana/sana/cli/run.py similarity index 100% rename from sana/sana_600M/packages/Sana/sana/cli/run.py rename to image/sana/sana-600m/packages/Sana/sana/cli/run.py diff --git a/sana/sana_600M/packages/Sana/sana/cli/upload2hf.py b/image/sana/sana-600m/packages/Sana/sana/cli/upload2hf.py similarity index 100% rename from sana/sana_600M/packages/Sana/sana/cli/upload2hf.py rename to image/sana/sana-600m/packages/Sana/sana/cli/upload2hf.py diff --git a/sana/sana_600M/packages/Sana/sana/tools/__init__.py b/image/sana/sana-600m/packages/Sana/sana/tools/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/sana/tools/__init__.py rename to image/sana/sana-600m/packages/Sana/sana/tools/__init__.py diff --git a/sana/sana_600M/packages/Sana/sana/tools/download.py b/image/sana/sana-600m/packages/Sana/sana/tools/download.py similarity index 100% rename from sana/sana_600M/packages/Sana/sana/tools/download.py rename to image/sana/sana-600m/packages/Sana/sana/tools/download.py diff --git a/sana/sana_600M/packages/Sana/sana/tools/hf_utils.py b/image/sana/sana-600m/packages/Sana/sana/tools/hf_utils.py similarity index 100% rename from sana/sana_600M/packages/Sana/sana/tools/hf_utils.py rename to image/sana/sana-600m/packages/Sana/sana/tools/hf_utils.py diff --git a/sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric.sh b/image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric.sh rename to image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric.sh diff --git a/sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric_dpg.sh b/image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric_dpg.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric_dpg.sh rename to image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric_dpg.sh diff --git a/sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric_geneval.sh b/image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric_geneval.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric_geneval.sh rename to image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric_geneval.sh diff --git a/sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh b/image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh rename to image/sana/sana-600m/packages/Sana/scripts/bash_run_inference_metric_imagereward.sh diff --git a/sana/sana_600M/packages/Sana/scripts/infer_metric_run_inference_metric.sh b/image/sana/sana-600m/packages/Sana/scripts/infer_metric_run_inference_metric.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/infer_metric_run_inference_metric.sh rename to image/sana/sana-600m/packages/Sana/scripts/infer_metric_run_inference_metric.sh diff --git a/sana/sana_600M/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh b/image/sana/sana-600m/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh rename to image/sana/sana-600m/packages/Sana/scripts/infer_metric_run_inference_metric_geneval.sh diff --git a/sana/sana_600M/packages/Sana/scripts/infer_run_inference.sh b/image/sana/sana-600m/packages/Sana/scripts/infer_run_inference.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/infer_run_inference.sh rename to image/sana/sana-600m/packages/Sana/scripts/infer_run_inference.sh diff --git a/sana/sana_600M/packages/Sana/scripts/infer_run_inference_geneval.sh b/image/sana/sana-600m/packages/Sana/scripts/infer_run_inference_geneval.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/infer_run_inference_geneval.sh rename to image/sana/sana-600m/packages/Sana/scripts/infer_run_inference_geneval.sh diff --git a/sana/sana_600M/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh b/image/sana/sana-600m/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh rename to image/sana/sana-600m/packages/Sana/scripts/infer_run_inference_geneval_diffusers.sh diff --git a/sana/sana_600M/packages/Sana/scripts/inference.py b/image/sana/sana-600m/packages/Sana/scripts/inference.py similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/inference.py rename to image/sana/sana-600m/packages/Sana/scripts/inference.py diff --git a/sana/sana_600M/packages/Sana/scripts/inference_dpg.py b/image/sana/sana-600m/packages/Sana/scripts/inference_dpg.py similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/inference_dpg.py rename to image/sana/sana-600m/packages/Sana/scripts/inference_dpg.py diff --git a/sana/sana_600M/packages/Sana/scripts/inference_geneval.py b/image/sana/sana-600m/packages/Sana/scripts/inference_geneval.py similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/inference_geneval.py rename to image/sana/sana-600m/packages/Sana/scripts/inference_geneval.py diff --git a/sana/sana_600M/packages/Sana/scripts/inference_geneval_diffusers.py b/image/sana/sana-600m/packages/Sana/scripts/inference_geneval_diffusers.py similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/inference_geneval_diffusers.py rename to image/sana/sana-600m/packages/Sana/scripts/inference_geneval_diffusers.py diff --git a/sana/sana_600M/packages/Sana/scripts/inference_image_reward.py b/image/sana/sana-600m/packages/Sana/scripts/inference_image_reward.py similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/inference_image_reward.py rename to image/sana/sana-600m/packages/Sana/scripts/inference_image_reward.py diff --git a/sana/sana_600M/packages/Sana/scripts/interface.py b/image/sana/sana-600m/packages/Sana/scripts/interface.py similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/interface.py rename to image/sana/sana-600m/packages/Sana/scripts/interface.py diff --git a/sana/sana_600M/packages/Sana/scripts/style.css b/image/sana/sana-600m/packages/Sana/scripts/style.css similarity index 100% rename from sana/sana_600M/packages/Sana/scripts/style.css rename to image/sana/sana-600m/packages/Sana/scripts/style.css diff --git a/sana/sana_600M/packages/Sana/tests/bash/entry.sh b/image/sana/sana-600m/packages/Sana/tests/bash/entry.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tests/bash/entry.sh rename to image/sana/sana-600m/packages/Sana/tests/bash/entry.sh diff --git a/sana/sana_600M/packages/Sana/tests/bash/test_inference.sh b/image/sana/sana-600m/packages/Sana/tests/bash/test_inference.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tests/bash/test_inference.sh rename to image/sana/sana-600m/packages/Sana/tests/bash/test_inference.sh diff --git a/sana/sana_600M/packages/Sana/tests/bash/test_training_1epoch.sh b/image/sana/sana-600m/packages/Sana/tests/bash/test_training_1epoch.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tests/bash/test_training_1epoch.sh rename to image/sana/sana-600m/packages/Sana/tests/bash/test_training_1epoch.sh diff --git a/llama/llama-7b/model/__init__.py b/image/sana/sana-600m/packages/Sana/tools/__init__.py similarity index 100% rename from llama/llama-7b/model/__init__.py rename to image/sana/sana-600m/packages/Sana/tools/__init__.py diff --git a/sana/sana_600M/packages/Sana/tools/convert_py_to_yaml.py b/image/sana/sana-600m/packages/Sana/tools/convert_py_to_yaml.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/convert_py_to_yaml.py rename to image/sana/sana-600m/packages/Sana/tools/convert_py_to_yaml.py diff --git a/sana/sana_600M/packages/Sana/tools/convert_sana_pag_to_diffusers.py b/image/sana/sana-600m/packages/Sana/tools/convert_sana_pag_to_diffusers.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/convert_sana_pag_to_diffusers.py rename to image/sana/sana-600m/packages/Sana/tools/convert_sana_pag_to_diffusers.py diff --git a/sana/sana_600M/packages/Sana/tools/convert_sana_to_diffusers.py b/image/sana/sana-600m/packages/Sana/tools/convert_sana_to_diffusers.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/convert_sana_to_diffusers.py rename to image/sana/sana-600m/packages/Sana/tools/convert_sana_to_diffusers.py diff --git a/sana/sana_600M/packages/Sana/tools/download.py b/image/sana/sana-600m/packages/Sana/tools/download.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/download.py rename to image/sana/sana-600m/packages/Sana/tools/download.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/.gitignore b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/.gitignore similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/.gitignore rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/.gitignore diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/LICENSE b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/LICENSE similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/LICENSE rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/LICENSE diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/README.md b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/README.md similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/README.md rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/README.md diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/clip_score.py b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/clip_score.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/clip_score.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/clip_score.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/setup.py b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/setup.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/setup.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/setup.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__init__.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/src/clip_score/__main__.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py b/image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/clip-score/src/clip_score/clip_score.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/compute_clipscore.sh b/image/sana/sana-600m/packages/Sana/tools/metrics/compute_clipscore.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/compute_clipscore.sh rename to image/sana/sana-600m/packages/Sana/tools/metrics/compute_clipscore.sh diff --git a/sana/sana_600M/packages/Sana/tools/metrics/compute_dpg.sh b/image/sana/sana-600m/packages/Sana/tools/metrics/compute_dpg.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/compute_dpg.sh rename to image/sana/sana-600m/packages/Sana/tools/metrics/compute_dpg.sh diff --git a/sana/sana_600M/packages/Sana/tools/metrics/compute_fid_embedding.sh b/image/sana/sana-600m/packages/Sana/tools/metrics/compute_fid_embedding.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/compute_fid_embedding.sh rename to image/sana/sana-600m/packages/Sana/tools/metrics/compute_fid_embedding.sh diff --git a/sana/sana_600M/packages/Sana/tools/metrics/compute_geneval.sh b/image/sana/sana-600m/packages/Sana/tools/metrics/compute_geneval.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/compute_geneval.sh rename to image/sana/sana-600m/packages/Sana/tools/metrics/compute_geneval.sh diff --git a/sana/sana_600M/packages/Sana/tools/metrics/compute_imagereward.sh b/image/sana/sana-600m/packages/Sana/tools/metrics/compute_imagereward.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/compute_imagereward.sh rename to image/sana/sana-600m/packages/Sana/tools/metrics/compute_imagereward.sh diff --git a/sana/sana_600M/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py b/image/sana/sana-600m/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/dpg_bench/compute_dpg_bench.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv b/image/sana/sana-600m/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv rename to image/sana/sana-600m/packages/Sana/tools/metrics/dpg_bench/dpg_bench.csv diff --git a/sana/sana_600M/packages/Sana/tools/metrics/dpg_bench/requirements.txt b/image/sana/sana-600m/packages/Sana/tools/metrics/dpg_bench/requirements.txt similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/dpg_bench/requirements.txt rename to image/sana/sana-600m/packages/Sana/tools/metrics/dpg_bench/requirements.txt diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/LICENSE b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/LICENSE similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/LICENSE rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/LICENSE diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/README.md b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/README.md similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/README.md rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/README.md diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/annotations_clip.csv diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/annotations_if-xl.csv diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/annotations_sdv2.csv diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/annotations/mturk_hit_template.html diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/environment.yml b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/environment.yml similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/environment.yml rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/environment.yml diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/download_models.sh diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/evaluate_images.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/object_names.txt diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/evaluation/summary_scores.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/generation/diffusers_generate.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/images/geneval_figure_1.png diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/create_prompts.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/evaluation_metadata.jsonl diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/generation_prompts.txt diff --git a/sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/object_names.txt b/image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/object_names.txt similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/geneval/prompts/object_names.txt rename to image/sana/sana-600m/packages/Sana/tools/metrics/geneval/prompts/object_names.txt diff --git a/sana/sana_600M/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json b/image/sana/sana-600m/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json rename to image/sana/sana-600m/packages/Sana/tools/metrics/image_reward/benchmark-prompts-dict.json diff --git a/sana/sana_600M/packages/Sana/tools/metrics/image_reward/compute_image_reward.py b/image/sana/sana-600m/packages/Sana/tools/metrics/image_reward/compute_image_reward.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/image_reward/compute_image_reward.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/image_reward/compute_image_reward.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/.gitignore b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/.gitignore similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/.gitignore rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/.gitignore diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/CHANGELOG.md diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/LICENSE b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/LICENSE similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/LICENSE rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/LICENSE diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/README.md b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/README.md similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/README.md rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/README.md diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/compute_fid.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/noxfile.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/noxfile.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/noxfile.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/noxfile.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/setup.cfg b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/setup.cfg similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/setup.cfg rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/setup.cfg diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/setup.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/setup.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/setup.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/setup.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__init__.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/__main__.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/fid_score.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/src/pytorch_fid/inception.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py b/image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/pytorch-fid/tests/test_fid_score.py diff --git a/sana/sana_600M/packages/Sana/tools/metrics/utils.py b/image/sana/sana-600m/packages/Sana/tools/metrics/utils.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/metrics/utils.py rename to image/sana/sana-600m/packages/Sana/tools/metrics/utils.py diff --git a/sana/sana_600M/packages/Sana/train_scripts/train.py b/image/sana/sana-600m/packages/Sana/train_scripts/train.py similarity index 100% rename from sana/sana_600M/packages/Sana/train_scripts/train.py rename to image/sana/sana-600m/packages/Sana/train_scripts/train.py diff --git a/sana/sana_600M/packages/Sana/train_scripts/train.sh b/image/sana/sana-600m/packages/Sana/train_scripts/train.sh similarity index 100% rename from sana/sana_600M/packages/Sana/train_scripts/train.sh rename to image/sana/sana-600m/packages/Sana/train_scripts/train.sh diff --git a/image/segment-anything/README.md b/image/segment-anything/README.md new file mode 100644 index 000000000..1f26a1276 --- /dev/null +++ b/image/segment-anything/README.md @@ -0,0 +1,31 @@ +# Segment Anything + +Deploy Segment Anything for image segmentation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image segmentation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "image_url": "https://as2.ftcdn.net/v2/jpg/00/66/26/87/1000_F_66268784_jccdcfdpf2vmq5X8raYA8JQT0sziZ1H9.jpg" +}' +``` + +## Configuration highlights + +- System packages: `python3-opencv` diff --git a/segment-anything/config.yaml b/image/segment-anything/config.yaml similarity index 88% rename from segment-anything/config.yaml rename to image/segment-anything/config.yaml index 05f1991f1..afea7f154 100644 --- a/segment-anything/config.yaml +++ b/image/segment-anything/config.yaml @@ -1,9 +1,11 @@ +description: "Segment Anything for image segmentation" environment_variables: {} external_data: - local_data_path: sam_vit_h_4b8939.pth url: https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth external_package_dirs: [] model_metadata: + repo_id: "facebook/sam-vit-huge" example_model_input: image_url: https://as2.ftcdn.net/v2/jpg/00/66/26/87/1000_F_66268784_jccdcfdpf2vmq5X8raYA8JQT0sziZ1H9.jpg model_name: Segment Anything diff --git a/llava/llava-1.6-sgl/model/__init__.py b/image/segment-anything/model/__init__.py similarity index 100% rename from llava/llava-1.6-sgl/model/__init__.py rename to image/segment-anything/model/__init__.py diff --git a/segment-anything/model/model.py b/image/segment-anything/model/model.py similarity index 100% rename from segment-anything/model/model.py rename to image/segment-anything/model/model.py diff --git a/image/stable-diffusion/dreamshaper-lcm/README.md b/image/stable-diffusion/dreamshaper-lcm/README.md new file mode 100644 index 000000000..3928394a5 --- /dev/null +++ b/image/stable-diffusion/dreamshaper-lcm/README.md @@ -0,0 +1,31 @@ +# Dreamshaper Latent Consistency Model + +Deploy Dreamshaper Latent Consistency Model for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/image/stable-diffusion/dreamshaper-lcm/config.yaml b/image/stable-diffusion/dreamshaper-lcm/config.yaml new file mode 100644 index 000000000..1b5ffd430 --- /dev/null +++ b/image/stable-diffusion/dreamshaper-lcm/config.yaml @@ -0,0 +1,21 @@ +description: "Dreamshaper Latent Consistency Model for image generation" +environment_variables: {} +external_package_dirs: [] +model_metadata: + repo_id: "SimianLuo/LCM_Dreamshaper_v7" + example_model_input: + prompt: A photo of an astronaut riding a horse +model_name: Dreamshaper Latent Consistency Model +python_version: py311 +requirements: +- diffusers==0.21.4 +- transformers==4.34.1 +- accelerate==0.23.0 +- torch==2.1.0 +resources: + accelerator: A10G + cpu: '1' + memory: 2Gi + use_gpu: true +secrets: {} +system_packages: [] diff --git a/llava/llava-v1.5-7b/model/__init__.py b/image/stable-diffusion/dreamshaper-lcm/model/__init__.py similarity index 100% rename from llava/llava-v1.5-7b/model/__init__.py rename to image/stable-diffusion/dreamshaper-lcm/model/__init__.py diff --git a/stable-diffusion/dreamshaper-lcm/model/model.py b/image/stable-diffusion/dreamshaper-lcm/model/model.py similarity index 100% rename from stable-diffusion/dreamshaper-lcm/model/model.py rename to image/stable-diffusion/dreamshaper-lcm/model/model.py diff --git a/image/stable-diffusion/playground-v2-trt/README.md b/image/stable-diffusion/playground-v2-trt/README.md new file mode 100644 index 000000000..99add8568 --- /dev/null +++ b/image/stable-diffusion/playground-v2-trt/README.md @@ -0,0 +1,35 @@ +# Playground v2 - TensorRT + +Generate original images from text prompts. + +| Property | Value | +|----------|-------| +| Model | [baseten/playground-v2-trt-8.6.1.post1-engine-A100](https://huggingface.co/baseten/playground-v2-trt-8.6.1.post1-engine-A100) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" +}' +``` + +## Configuration highlights + +- Base image: `nvcr.io/nvidia/pytorch:23.11-py3` +- Predict concurrency: **1** +- System packages: `python3.10-venv, ffmpeg, libsm6, libxext6` +- Environment variables: `HF_HUB_ENABLE_HF_TRANSFER` diff --git a/stable-diffusion/playground-v2-trt/config.yaml b/image/stable-diffusion/playground-v2-trt/config.yaml similarity index 85% rename from stable-diffusion/playground-v2-trt/config.yaml rename to image/stable-diffusion/playground-v2-trt/config.yaml index 380979254..d7197438e 100644 --- a/stable-diffusion/playground-v2-trt/config.yaml +++ b/image/stable-diffusion/playground-v2-trt/config.yaml @@ -12,11 +12,13 @@ model_cache: - config.json - diffusion_pytorch_model.safetensors repo_id: madebyollin/sdxl-vae-fp16-fix + use_volume: false - allow_patterns: - '*.json' - '*.fp16.safetensors' - playground-v2.safetensors repo_id: playgroundai/playground-v2-1024px-aesthetic + use_volume: false model_metadata: example_model_input: prompt: Astronaut in a jungle, cold color palette, muted colors, detailed, 8k @@ -26,20 +28,20 @@ model_metadata: model_name: Playground v2 - TensorRT python_version: py39 requirements: -- accelerate -- colored -- cuda-python -- ftfy -- nvtx +- accelerate==0.25.0 +- colored==2.2.4 +- cuda-python==12.3.0 +- ftfy==6.1.3 +- nvtx==0.2.10 - opencv-python==4.8.0.74 -- scipy +- scipy==1.11.4 - transformers==4.31.0 -- safetensors -- hf_transfer +- safetensors==0.4.1 +- hf_transfer==0.1.4 - diffusers==0.23.1 - invisible-watermark>=0.2.0 - --extra-index-url https://pypi.ngc.nvidia.com -- polygraphy +- polygraphy==0.49.9 - --extra-index-url https://pypi.nvidia.com - tensorrt==8.6.1.post1 resources: diff --git a/llava/llava-v1.5-7b/packages/llava/serve/__init__.py b/image/stable-diffusion/playground-v2-trt/model/__init__.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/__init__.py rename to image/stable-diffusion/playground-v2-trt/model/__init__.py diff --git a/stable-diffusion/playground-v2-trt/model/model.py b/image/stable-diffusion/playground-v2-trt/model/model.py similarity index 100% rename from stable-diffusion/playground-v2-trt/model/model.py rename to image/stable-diffusion/playground-v2-trt/model/model.py diff --git a/stable-diffusion/playground-v2-trt/packages/diffusion/trtclip.py b/image/stable-diffusion/playground-v2-trt/packages/diffusion/trtclip.py similarity index 100% rename from stable-diffusion/playground-v2-trt/packages/diffusion/trtclip.py rename to image/stable-diffusion/playground-v2-trt/packages/diffusion/trtclip.py diff --git a/stable-diffusion/playground-v2-trt/packages/diffusion/trtunet.py b/image/stable-diffusion/playground-v2-trt/packages/diffusion/trtunet.py similarity index 100% rename from stable-diffusion/playground-v2-trt/packages/diffusion/trtunet.py rename to image/stable-diffusion/playground-v2-trt/packages/diffusion/trtunet.py diff --git a/stable-diffusion/playground-v2-trt/show.py b/image/stable-diffusion/playground-v2-trt/show.py similarity index 100% rename from stable-diffusion/playground-v2-trt/show.py rename to image/stable-diffusion/playground-v2-trt/show.py diff --git a/image/stable-diffusion/sd-textual-inversion/README.md b/image/stable-diffusion/sd-textual-inversion/README.md new file mode 100644 index 000000000..5794c95b4 --- /dev/null +++ b/image/stable-diffusion/sd-textual-inversion/README.md @@ -0,0 +1,31 @@ +# SD_Textual_Inversion + +Deploy SD_Textual_Inversion for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/stable-diffusion/sd-textual-inversion/config.yaml b/image/stable-diffusion/sd-textual-inversion/config.yaml similarity index 51% rename from stable-diffusion/sd-textual-inversion/config.yaml rename to image/stable-diffusion/sd-textual-inversion/config.yaml index 2008f9d2f..1570d85c0 100644 --- a/stable-diffusion/sd-textual-inversion/config.yaml +++ b/image/stable-diffusion/sd-textual-inversion/config.yaml @@ -1,6 +1,10 @@ +description: "Stable Diffusion with Textual Inversion for image generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "SG161222/Realistic_Vision_V4.0_noVAE" + example_model_input: + prompt: A photo of an astronaut riding a horse in the style of LulaCipher pretty_name: Stable Diffusion - Textual Inversion tags: - image-generation @@ -8,11 +12,11 @@ model_name: SD_Textual_Inversion python_version: py311 requirements: - diffusers==0.16.1 -- transformers -- ftfy -- accelerate -- torch -- pillow +- transformers==4.36.0 +- ftfy==6.1.3 +- accelerate==0.25.0 +- torch==2.1.0 +- pillow==10.1.0 resources: accelerator: T4 cpu: 500m diff --git a/stable-diffusion/sd-textual-inversion/data/LulaCipher.bin b/image/stable-diffusion/sd-textual-inversion/data/LulaCipher.bin similarity index 100% rename from stable-diffusion/sd-textual-inversion/data/LulaCipher.bin rename to image/stable-diffusion/sd-textual-inversion/data/LulaCipher.bin diff --git a/llava/llava-v1.6-34b/model/__init__.py b/image/stable-diffusion/sd-textual-inversion/model/__init__.py similarity index 100% rename from llava/llava-v1.6-34b/model/__init__.py rename to image/stable-diffusion/sd-textual-inversion/model/__init__.py diff --git a/stable-diffusion/sd-textual-inversion/model/model.py b/image/stable-diffusion/sd-textual-inversion/model/model.py similarity index 100% rename from stable-diffusion/sd-textual-inversion/model/model.py rename to image/stable-diffusion/sd-textual-inversion/model/model.py diff --git a/image/stable-diffusion/sd-turbo/README.md b/image/stable-diffusion/sd-turbo/README.md new file mode 100644 index 000000000..1372ab9ec --- /dev/null +++ b/image/stable-diffusion/sd-turbo/README.md @@ -0,0 +1,32 @@ +# SD Turbo + +Deploy [stabilityai/sdxl-turbo](https://huggingface.co/stabilityai/sdxl-turbo) for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Model | [stabilityai/sdxl-turbo](https://huggingface.co/stabilityai/sdxl-turbo) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A tree in a field under the night sky" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/stable-diffusion/sd-turbo/config.yaml b/image/stable-diffusion/sd-turbo/config.yaml similarity index 91% rename from stable-diffusion/sd-turbo/config.yaml rename to image/stable-diffusion/sd-turbo/config.yaml index 63469a4a4..e9ffc0152 100644 --- a/stable-diffusion/sd-turbo/config.yaml +++ b/image/stable-diffusion/sd-turbo/config.yaml @@ -1,3 +1,4 @@ +description: "stabilityai/sdxl-turbo for image generation" environment_variables: {} external_package_dirs: [] model_cache: diff --git a/magic-animate/model/__init__.py b/image/stable-diffusion/sd-turbo/model/__init__.py similarity index 100% rename from magic-animate/model/__init__.py rename to image/stable-diffusion/sd-turbo/model/__init__.py diff --git a/stable-diffusion/sd-turbo/model/model.py b/image/stable-diffusion/sd-turbo/model/model.py similarity index 100% rename from stable-diffusion/sd-turbo/model/model.py rename to image/stable-diffusion/sd-turbo/model/model.py diff --git a/image/stable-diffusion/sdxl-controlnet-canny/README.md b/image/stable-diffusion/sdxl-controlnet-canny/README.md new file mode 100644 index 000000000..de0fd7cbe --- /dev/null +++ b/image/stable-diffusion/sdxl-controlnet-canny/README.md @@ -0,0 +1,31 @@ +# SDXL ControlNet Canny + +Deploy SDXL ControlNet Canny for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G:2 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/stable-diffusion/sdxl-controlnet-canny/baseten-logo.gif b/image/stable-diffusion/sdxl-controlnet-canny/baseten-logo.gif similarity index 100% rename from stable-diffusion/sdxl-controlnet-canny/baseten-logo.gif rename to image/stable-diffusion/sdxl-controlnet-canny/baseten-logo.gif diff --git a/stable-diffusion/sdxl-controlnet-canny/config.yaml b/image/stable-diffusion/sdxl-controlnet-canny/config.yaml similarity index 82% rename from stable-diffusion/sdxl-controlnet-canny/config.yaml rename to image/stable-diffusion/sdxl-controlnet-canny/config.yaml index a6acda3f9..cd246fbf6 100644 --- a/stable-diffusion/sdxl-controlnet-canny/config.yaml +++ b/image/stable-diffusion/sdxl-controlnet-canny/config.yaml @@ -1,11 +1,14 @@ +description: "SDXL ControlNet Canny for image generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "stabilityai/stable-diffusion-xl-base-1.0" avatar_url: https://cdn.baseten.co/production/static/stability.png cover_image_url: https://cdn.baseten.co/production/static/sd.png example_model_input: prompt: aerial view, a futuristic research complex in a bright foggy jungle, hard lighting + image: "" model_metadata: null pretty_name: Stable Diffusion ControlNet tags: diff --git a/metavoice-1b/model/__init__.py b/image/stable-diffusion/sdxl-controlnet-canny/model/__init__.py similarity index 100% rename from metavoice-1b/model/__init__.py rename to image/stable-diffusion/sdxl-controlnet-canny/model/__init__.py diff --git a/stable-diffusion/sdxl-controlnet-canny/model/model.py b/image/stable-diffusion/sdxl-controlnet-canny/model/model.py similarity index 100% rename from stable-diffusion/sdxl-controlnet-canny/model/model.py rename to image/stable-diffusion/sdxl-controlnet-canny/model/model.py diff --git a/image/stable-diffusion/sdxl-controlnet-depth/README.md b/image/stable-diffusion/sdxl-controlnet-depth/README.md new file mode 100644 index 000000000..48a775323 --- /dev/null +++ b/image/stable-diffusion/sdxl-controlnet-depth/README.md @@ -0,0 +1,31 @@ +# SDXL ControlNet Depth + +Deploy SDXL ControlNet Depth for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G:2 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "large bed, abstract painting on the wall, fluffy rug on the floor, ambient lighting, extremely detailed" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/stable-diffusion/sdxl-controlnet-depth/config.yaml b/image/stable-diffusion/sdxl-controlnet-depth/config.yaml similarity index 82% rename from stable-diffusion/sdxl-controlnet-depth/config.yaml rename to image/stable-diffusion/sdxl-controlnet-depth/config.yaml index b0ca0d71b..92cd2829d 100644 --- a/stable-diffusion/sdxl-controlnet-depth/config.yaml +++ b/image/stable-diffusion/sdxl-controlnet-depth/config.yaml @@ -1,11 +1,14 @@ +description: "SDXL ControlNet Depth for image generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "stabilityai/stable-diffusion-xl-base-1.0" avatar_url: https://cdn.baseten.co/production/static/stability.png cover_image_url: https://cdn.baseten.co/production/static/sd.png example_model_input: prompt: large bed, abstract painting on the wall, fluffy rug on the floor, ambient lighting, extremely detailed + image: "" pretty_name: Stable Diffusion ControlNet Depth tags: - image-generation diff --git a/midnight/model/__init__.py b/image/stable-diffusion/sdxl-controlnet-depth/model/__init__.py similarity index 100% rename from midnight/model/__init__.py rename to image/stable-diffusion/sdxl-controlnet-depth/model/__init__.py diff --git a/stable-diffusion/sdxl-controlnet-depth/model/model.py b/image/stable-diffusion/sdxl-controlnet-depth/model/model.py similarity index 100% rename from stable-diffusion/sdxl-controlnet-depth/model/model.py rename to image/stable-diffusion/sdxl-controlnet-depth/model/model.py diff --git a/image/stable-diffusion/sdxl-controlnet/README.md b/image/stable-diffusion/sdxl-controlnet/README.md new file mode 100644 index 000000000..c7847277a --- /dev/null +++ b/image/stable-diffusion/sdxl-controlnet/README.md @@ -0,0 +1,31 @@ +# SDXL ControlNet + +Deploy SDXL ControlNet for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/stable-diffusion/sdxl-controlnet/baseten-logo.gif b/image/stable-diffusion/sdxl-controlnet/baseten-logo.gif similarity index 100% rename from stable-diffusion/sdxl-controlnet/baseten-logo.gif rename to image/stable-diffusion/sdxl-controlnet/baseten-logo.gif diff --git a/stable-diffusion/sdxl-controlnet/config.yaml b/image/stable-diffusion/sdxl-controlnet/config.yaml similarity index 69% rename from stable-diffusion/sdxl-controlnet/config.yaml rename to image/stable-diffusion/sdxl-controlnet/config.yaml index 3ef4081ae..11067692e 100644 --- a/stable-diffusion/sdxl-controlnet/config.yaml +++ b/image/stable-diffusion/sdxl-controlnet/config.yaml @@ -1,22 +1,25 @@ +description: "SDXL ControlNet for image generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "stabilityai/stable-diffusion-xl-base-1.0" avatar_url: https://cdn.baseten.co/production/static/stability.png cover_image_url: https://cdn.baseten.co/production/static/sd.png example_model_input: prompt: aerial view, a futuristic research complex in a bright foggy jungle, hard lighting + image: "" pretty_name: Stable Diffusion ControlNet tags: - image-generation model_name: SDXL ControlNet python_version: py39 requirements: -- accelerate -- transformers -- safetensors -- opencv-python -- diffusers +- accelerate==0.25.0 +- transformers==4.36.0 +- safetensors==0.4.1 +- opencv-python==4.8.1.78 +- diffusers==0.24.0 resources: accelerator: A10G cpu: 3500m diff --git a/mistral/mistral-7b-chat/model/__init__.py b/image/stable-diffusion/sdxl-controlnet/model/__init__.py similarity index 100% rename from mistral/mistral-7b-chat/model/__init__.py rename to image/stable-diffusion/sdxl-controlnet/model/__init__.py diff --git a/stable-diffusion/sdxl-controlnet/model/model.py b/image/stable-diffusion/sdxl-controlnet/model/model.py similarity index 100% rename from stable-diffusion/sdxl-controlnet/model/model.py rename to image/stable-diffusion/sdxl-controlnet/model/model.py diff --git a/image/stable-diffusion/sdxl-lightning/README.md b/image/stable-diffusion/sdxl-lightning/README.md new file mode 100644 index 000000000..44d42fc42 --- /dev/null +++ b/image/stable-diffusion/sdxl-lightning/README.md @@ -0,0 +1,31 @@ +# SDXL Lightning + +Deploy SDXL Lightning for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A tree in a field under the night sky" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/stable-diffusion/sdxl-lightning/config.yaml b/image/stable-diffusion/sdxl-lightning/config.yaml similarity index 87% rename from stable-diffusion/sdxl-lightning/config.yaml rename to image/stable-diffusion/sdxl-lightning/config.yaml index 847f14942..6154693a1 100644 --- a/stable-diffusion/sdxl-lightning/config.yaml +++ b/image/stable-diffusion/sdxl-lightning/config.yaml @@ -1,6 +1,8 @@ +description: "SDXL Lightning for image generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "ByteDance/SDXL-Lightning" avatar_url: https://cdn.baseten.co/production/static/stability.png cover_image_url: https://cdn.baseten.co/production/static/sd.png example_model_input: diff --git a/mistral/mistral-7b-instruct-vllm/model/__init__.py b/image/stable-diffusion/sdxl-lightning/model/__init__.py similarity index 100% rename from mistral/mistral-7b-instruct-vllm/model/__init__.py rename to image/stable-diffusion/sdxl-lightning/model/__init__.py diff --git a/stable-diffusion/sdxl-lightning/model/model.py b/image/stable-diffusion/sdxl-lightning/model/model.py similarity index 100% rename from stable-diffusion/sdxl-lightning/model/model.py rename to image/stable-diffusion/sdxl-lightning/model/model.py diff --git a/image/stable-diffusion/sdxl-lora-swapping/README.md b/image/stable-diffusion/sdxl-lora-swapping/README.md new file mode 100644 index 000000000..59a134a77 --- /dev/null +++ b/image/stable-diffusion/sdxl-lora-swapping/README.md @@ -0,0 +1,35 @@ +# Stable Diffusion XL with LoRA Swapping + +Deploy Stable Diffusion XL with LoRA Swapping for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "lora": { + "repo_id": "nerijs/pixel-art-xl", + "weights": "pixel-art-xl.safetensors" + }, + "prompt": "pixel art, an baby giraffe" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/stable-diffusion/sdxl-lora-swapping/config.yaml b/image/stable-diffusion/sdxl-lora-swapping/config.yaml similarity index 81% rename from stable-diffusion/sdxl-lora-swapping/config.yaml rename to image/stable-diffusion/sdxl-lora-swapping/config.yaml index e78e10fba..5aae62310 100644 --- a/stable-diffusion/sdxl-lora-swapping/config.yaml +++ b/image/stable-diffusion/sdxl-lora-swapping/config.yaml @@ -1,6 +1,8 @@ +description: "Stable Diffusion XL with LoRA Swapping for image generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: stabilityai/stable-diffusion-xl-base-1.0 example_model_input: lora: repo_id: nerijs/pixel-art-xl diff --git a/mistral/mistral-7b-instruct/model/__init__.py b/image/stable-diffusion/sdxl-lora-swapping/model/__init__.py similarity index 100% rename from mistral/mistral-7b-instruct/model/__init__.py rename to image/stable-diffusion/sdxl-lora-swapping/model/__init__.py diff --git a/stable-diffusion/sdxl-lora-swapping/model/model.py b/image/stable-diffusion/sdxl-lora-swapping/model/model.py similarity index 100% rename from stable-diffusion/sdxl-lora-swapping/model/model.py rename to image/stable-diffusion/sdxl-lora-swapping/model/model.py diff --git a/image/stable-diffusion/sdxl-lora/README.md b/image/stable-diffusion/sdxl-lora/README.md new file mode 100644 index 000000000..d7f527625 --- /dev/null +++ b/image/stable-diffusion/sdxl-lora/README.md @@ -0,0 +1,31 @@ +# Stable Diffusion XL with LoRA + +Deploy Stable Diffusion XL with LoRA for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/image/stable-diffusion/sdxl-lora/config.yaml b/image/stable-diffusion/sdxl-lora/config.yaml new file mode 100644 index 000000000..717160710 --- /dev/null +++ b/image/stable-diffusion/sdxl-lora/config.yaml @@ -0,0 +1,25 @@ +description: "Stable Diffusion XL with LoRA for image generation" +environment_variables: {} +external_package_dirs: [] +model_metadata: + repo_id: "stabilityai/stable-diffusion-xl-base-1.0" + example_model_input: + prompt: A photo of an astronaut riding a horse +model_name: Stable Diffusion XL with LoRA +python_version: py311 +requirements: +- accelerate==0.25.0 +- transformers==4.36.0 +- safetensors==0.4.1 +- opencv-python==4.8.1.78 +- diffusers==0.24.0 +resources: + accelerator: A10G + cpu: 3500m + memory: 20Gi + use_gpu: true +secrets: {} +system_packages: +- ffmpeg +- libsm6 +- libxext6 diff --git a/mistral/mistral-7b/model/__init__.py b/image/stable-diffusion/sdxl-lora/model/__init__.py similarity index 100% rename from mistral/mistral-7b/model/__init__.py rename to image/stable-diffusion/sdxl-lora/model/__init__.py diff --git a/stable-diffusion/sdxl-lora/model/model.py b/image/stable-diffusion/sdxl-lora/model/model.py similarity index 100% rename from stable-diffusion/sdxl-lora/model/model.py rename to image/stable-diffusion/sdxl-lora/model/model.py diff --git a/image/stable-diffusion/sdxl-turbo/README.md b/image/stable-diffusion/sdxl-turbo/README.md new file mode 100644 index 000000000..b8b3bc107 --- /dev/null +++ b/image/stable-diffusion/sdxl-turbo/README.md @@ -0,0 +1,32 @@ +# SDXL Turbo + +Deploy [stabilityai/sdxl-turbo](https://huggingface.co/stabilityai/sdxl-turbo) for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Model | [stabilityai/sdxl-turbo](https://huggingface.co/stabilityai/sdxl-turbo) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A tree in a field under the night sky" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/stable-diffusion/sdxl-turbo/config.yaml b/image/stable-diffusion/sdxl-turbo/config.yaml similarity index 92% rename from stable-diffusion/sdxl-turbo/config.yaml rename to image/stable-diffusion/sdxl-turbo/config.yaml index d9fded05b..8e2e52dc3 100644 --- a/stable-diffusion/sdxl-turbo/config.yaml +++ b/image/stable-diffusion/sdxl-turbo/config.yaml @@ -1,3 +1,4 @@ +description: "stabilityai/sdxl-turbo for image generation" environment_variables: {} external_package_dirs: [] model_cache: diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/model/__init__.py b/image/stable-diffusion/sdxl-turbo/model/__init__.py similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/model/__init__.py rename to image/stable-diffusion/sdxl-turbo/model/__init__.py diff --git a/stable-diffusion/sdxl-turbo/model/model.py b/image/stable-diffusion/sdxl-turbo/model/model.py similarity index 100% rename from stable-diffusion/sdxl-turbo/model/model.py rename to image/stable-diffusion/sdxl-turbo/model/model.py diff --git a/image/stable-diffusion/stable-diffusion-3-medium/README.md b/image/stable-diffusion/stable-diffusion-3-medium/README.md new file mode 100644 index 000000000..cff014088 --- /dev/null +++ b/image/stable-diffusion/stable-diffusion-3-medium/README.md @@ -0,0 +1,35 @@ +# Stable Diffusion 3 Medium + +Deploy [stabilityai/stable-diffusion-3-medium-diffusers](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers) for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Model | [stabilityai/stable-diffusion-3-medium-diffusers](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py310 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` +- Environment variables: `HF_HUB_OFFLINE` diff --git a/stable-diffusion/stable-diffusion-3-medium/config.yaml b/image/stable-diffusion/stable-diffusion-3-medium/config.yaml similarity index 59% rename from stable-diffusion/stable-diffusion-3-medium/config.yaml rename to image/stable-diffusion/stable-diffusion-3-medium/config.yaml index d01dfd423..28d973a95 100644 --- a/stable-diffusion/stable-diffusion-3-medium/config.yaml +++ b/image/stable-diffusion/stable-diffusion-3-medium/config.yaml @@ -1,7 +1,10 @@ +description: "stabilityai/stable-diffusion-3-medium-diffusers for image generation" environment_variables: HF_HUB_OFFLINE: 1 external_package_dirs: [] -model_metadata: {} +model_metadata: + example_model_input: + prompt: A photo of an astronaut riding a horse model_cache: - repo_id: stabilityai/stable-diffusion-3-medium-diffusers use_volume: false @@ -9,10 +12,10 @@ model_name: Stable Diffusion 3 Medium python_version: py310 requirements: - diffusers==0.29.0 - - transformers - - accelerate - - sentencepiece - - protobuf + - transformers==4.36.0 + - accelerate==0.25.0 + - sentencepiece==0.1.99 + - protobuf==4.25.1 resources: accelerator: A100 use_gpu: true diff --git a/mistral/mixtral-8x22b/model/__init__.py b/image/stable-diffusion/stable-diffusion-3-medium/model/__init__.py similarity index 100% rename from mistral/mixtral-8x22b/model/__init__.py rename to image/stable-diffusion/stable-diffusion-3-medium/model/__init__.py diff --git a/stable-diffusion/stable-diffusion-3-medium/model/model.py b/image/stable-diffusion/stable-diffusion-3-medium/model/model.py similarity index 100% rename from stable-diffusion/stable-diffusion-3-medium/model/model.py rename to image/stable-diffusion/stable-diffusion-3-medium/model/model.py diff --git a/image/stable-diffusion/stable-diffusion-inpainting-trt/README.md b/image/stable-diffusion/stable-diffusion-inpainting-trt/README.md new file mode 100644 index 000000000..dfec6f6c2 --- /dev/null +++ b/image/stable-diffusion/stable-diffusion-inpainting-trt/README.md @@ -0,0 +1,31 @@ +# Stable Diffusion Inpainting TRT + +Deploy Stable Diffusion Inpainting TRT for image generation on Baseten. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/image/stable-diffusion/stable-diffusion-inpainting-trt/config.yaml b/image/stable-diffusion/stable-diffusion-inpainting-trt/config.yaml new file mode 100644 index 000000000..05ef54a83 --- /dev/null +++ b/image/stable-diffusion/stable-diffusion-inpainting-trt/config.yaml @@ -0,0 +1,18 @@ +description: "Stable Diffusion Inpainting with TensorRT for image inpainting" +environment_variables: {} +external_package_dirs: [] +model_metadata: + repo_id: "runwayml/stable-diffusion-inpainting" + example_model_input: + prompt: A beautiful vase of flowers on a table + image: "" + mask: "" +model_name: Stable Diffusion Inpainting TRT +python_version: py310 +requirements: [] +requirements_file: requirements.txt +resources: + accelerator: A10G + use_gpu: true +secrets: {} +system_packages: [] diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/model/__init__.py b/image/stable-diffusion/stable-diffusion-inpainting-trt/model/__init__.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/model/__init__.py rename to image/stable-diffusion/stable-diffusion-inpainting-trt/model/__init__.py diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/model/model.py b/image/stable-diffusion/stable-diffusion-inpainting-trt/model/model.py similarity index 100% rename from stable-diffusion/stable-diffusion-inpainting-trt/model/model.py rename to image/stable-diffusion/stable-diffusion-inpainting-trt/model/model.py diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/inpaint_pipeline.py b/image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/inpaint_pipeline.py similarity index 100% rename from stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/inpaint_pipeline.py rename to image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/inpaint_pipeline.py diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/models.py b/image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/models.py similarity index 100% rename from stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/models.py rename to image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/models.py diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/stable_diffusion_pipeline.py b/image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/stable_diffusion_pipeline.py similarity index 100% rename from stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/stable_diffusion_pipeline.py rename to image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/stable_diffusion_pipeline.py diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/utilities.py b/image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/utilities.py similarity index 100% rename from stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/utilities.py rename to image/stable-diffusion/stable-diffusion-inpainting-trt/packages/helpers/utilities.py diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/requirements.txt b/image/stable-diffusion/stable-diffusion-inpainting-trt/requirements.txt similarity index 100% rename from stable-diffusion/stable-diffusion-inpainting-trt/requirements.txt rename to image/stable-diffusion/stable-diffusion-inpainting-trt/requirements.txt diff --git a/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md new file mode 100644 index 000000000..184be9ec3 --- /dev/null +++ b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md @@ -0,0 +1,35 @@ +# Stable Diffusion XL - TensorRT + +Generate original images from text prompts. + +| Property | Value | +|----------|-------| +| Model | [baseten/sdxl-1.0-trt-8.6.1.post1-engine-H100](https://huggingface.co/baseten/sdxl-1.0-trt-8.6.1.post1-engine-H100) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | H100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" +}' +``` + +## Configuration highlights + +- Base image: `nvcr.io/nvidia/pytorch:23.11-py3` +- Predict concurrency: **1** +- System packages: `python3.10-venv, ffmpeg, libsm6, libxext6` +- Environment variables: `HF_HUB_ENABLE_HF_TRANSFER` diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/config.yaml b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/config.yaml similarity index 86% rename from stable-diffusion/stable-diffusion-xl-1.0-trt-h100/config.yaml rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/config.yaml index c7608fd76..a89232fa8 100644 --- a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/config.yaml +++ b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/config.yaml @@ -12,16 +12,19 @@ model_cache: - config.json - diffusion_pytorch_model.safetensors repo_id: madebyollin/sdxl-vae-fp16-fix + use_volume: false - allow_patterns: - '*.json' - '*.fp16.safetensors' - sd_xl_base_1.0.safetensors repo_id: stabilityai/stable-diffusion-xl-base-1.0 + use_volume: false - allow_patterns: - '*.json' - '*.fp16.safetensors' - sd_xl_refiner_1.0.safetensors repo_id: stabilityai/stable-diffusion-xl-refiner-1.0 + use_volume: false model_metadata: avatar_url: https://cdn.baseten.co/production/static/stability.png cover_image_url: https://cdn.baseten.co/production/static/sd.png @@ -33,20 +36,20 @@ model_metadata: model_name: Stable Diffusion XL - TensorRT python_version: py39 requirements: -- accelerate -- colored -- cuda-python -- ftfy -- nvtx +- accelerate==0.25.0 +- colored==2.2.4 +- cuda-python==12.3.0 +- ftfy==6.1.3 +- nvtx==0.2.10 - opencv-python==4.8.0.74 -- scipy +- scipy==1.11.4 - transformers==4.31.0 -- safetensors -- hf_transfer +- safetensors==0.4.1 +- hf_transfer==0.1.4 - diffusers==0.23.1 - invisible-watermark>=0.2.0 - --extra-index-url https://pypi.ngc.nvidia.com -- polygraphy +- polygraphy==0.49.9 - --extra-index-url https://pypi.nvidia.com - tensorrt==8.6.1.post1 resources: diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/__init__.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/__init__.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/__init__.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/__init__.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/model.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/model.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/model.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/model.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtclip.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtclip.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtclip.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtclip.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtunet.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtunet.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtunet.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/packages/diffusion/trtunet.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/show.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/show.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt-h100/show.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/show.py diff --git a/image/stable-diffusion/stable-diffusion-xl-1.0-trt/README.md b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/README.md new file mode 100644 index 000000000..9a7b6dd87 --- /dev/null +++ b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/README.md @@ -0,0 +1,35 @@ +# Stable Diffusion XL - TensorRT + +Generate original images from text prompts. + +| Property | Value | +|----------|-------| +| Model | [baseten/sdxl-1.0-trt-8.6.1.post1-engine](https://huggingface.co/baseten/sdxl-1.0-trt-8.6.1.post1-engine) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" +}' +``` + +## Configuration highlights + +- Base image: `nvcr.io/nvidia/pytorch:23.11-py3` +- Predict concurrency: **1** +- System packages: `python3.10-venv, ffmpeg, libsm6, libxext6` +- Environment variables: `HF_HUB_ENABLE_HF_TRANSFER` diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt/config.yaml b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/config.yaml similarity index 86% rename from stable-diffusion/stable-diffusion-xl-1.0-trt/config.yaml rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt/config.yaml index facf6a925..ba99ec3bf 100644 --- a/stable-diffusion/stable-diffusion-xl-1.0-trt/config.yaml +++ b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/config.yaml @@ -12,16 +12,19 @@ model_cache: - config.json - diffusion_pytorch_model.safetensors repo_id: madebyollin/sdxl-vae-fp16-fix + use_volume: false - allow_patterns: - '*.json' - '*.fp16.safetensors' - sd_xl_base_1.0.safetensors repo_id: stabilityai/stable-diffusion-xl-base-1.0 + use_volume: false - allow_patterns: - '*.json' - '*.fp16.safetensors' - sd_xl_refiner_1.0.safetensors repo_id: stabilityai/stable-diffusion-xl-refiner-1.0 + use_volume: false model_metadata: avatar_url: https://cdn.baseten.co/production/static/stability.png cover_image_url: https://cdn.baseten.co/production/static/sd.png @@ -33,20 +36,20 @@ model_metadata: model_name: Stable Diffusion XL - TensorRT python_version: py39 requirements: -- accelerate -- colored -- cuda-python -- ftfy -- nvtx +- accelerate==0.25.0 +- colored==2.2.4 +- cuda-python==12.3.0 +- ftfy==6.1.3 +- nvtx==0.2.10 - opencv-python==4.8.0.74 -- scipy +- scipy==1.11.4 - transformers==4.31.0 -- safetensors -- hf_transfer +- safetensors==0.4.1 +- hf_transfer==0.1.4 - diffusers==0.23.1 - invisible-watermark>=0.2.0 - --extra-index-url https://pypi.ngc.nvidia.com -- polygraphy +- polygraphy==0.49.9 - --extra-index-url https://pypi.nvidia.com - tensorrt==8.6.1.post1 resources: diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/__init__.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/model/__init__.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/__init__.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt/model/__init__.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt/model/model.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/model/model.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt/model/model.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt/model/model.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtclip.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtclip.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtclip.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtclip.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtunet.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtunet.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtunet.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt/packages/diffusion/trtunet.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt/show.py b/image/stable-diffusion/stable-diffusion-xl-1.0-trt/show.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt/show.py rename to image/stable-diffusion/stable-diffusion-xl-1.0-trt/show.py diff --git a/image/stable-diffusion/stable-diffusion-xl-1.0/README.md b/image/stable-diffusion/stable-diffusion-xl-1.0/README.md new file mode 100644 index 000000000..eecc45bcd --- /dev/null +++ b/image/stable-diffusion/stable-diffusion-xl-1.0/README.md @@ -0,0 +1,33 @@ +# Stable Diffusion XL + +Generate original images from text prompts. + +| Property | Value | +|----------|-------| +| Model | [madebyollin/sdxl-vae-fp16-fix](https://huggingface.co/madebyollin/sdxl-vae-fp16-fix) | +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A tree in a field under the night sky", + "use_refiner": true +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/stable-diffusion/stable-diffusion-xl-1.0/config.yaml b/image/stable-diffusion/stable-diffusion-xl-1.0/config.yaml similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0/config.yaml rename to image/stable-diffusion/stable-diffusion-xl-1.0/config.yaml diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/model/__init__.py b/image/stable-diffusion/stable-diffusion-xl-1.0/model/__init__.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/model/__init__.py rename to image/stable-diffusion/stable-diffusion-xl-1.0/model/__init__.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0/model/model.py b/image/stable-diffusion/stable-diffusion-xl-1.0/model/model.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0/model/model.py rename to image/stable-diffusion/stable-diffusion-xl-1.0/model/model.py diff --git a/stable-diffusion/stable-diffusion-xl-1.0/show.py b/image/stable-diffusion/stable-diffusion-xl-1.0/show.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0/show.py rename to image/stable-diffusion/stable-diffusion-xl-1.0/show.py diff --git a/image/stable-diffusion/stable-diffusion/README.md b/image/stable-diffusion/stable-diffusion/README.md new file mode 100644 index 000000000..b2de1c4bd --- /dev/null +++ b/image/stable-diffusion/stable-diffusion/README.md @@ -0,0 +1,31 @@ +# Stable Diffusion + +Generate original images from text prompts. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/stable-diffusion/stable-diffusion/config.yaml b/image/stable-diffusion/stable-diffusion/config.yaml similarity index 79% rename from stable-diffusion/stable-diffusion/config.yaml rename to image/stable-diffusion/stable-diffusion/config.yaml index 208e248e8..862e75187 100644 --- a/stable-diffusion/stable-diffusion/config.yaml +++ b/image/stable-diffusion/stable-diffusion/config.yaml @@ -9,25 +9,28 @@ external_data: url: https://baseten-public.s3.us-west-2.amazonaws.com/models/stable-diffusion-truss/vae/diffusion_pytorch_model.bin external_package_dirs: [] model_metadata: + repo_id: "stabilityai/stable-diffusion-2-1-base" avatar_url: https://cdn.baseten.co/production/static/stability.png cover_image_url: https://cdn.baseten.co/production/static/sd.png + example_model_input: + prompt: A photo of an astronaut riding a horse pretty_name: Stable Diffusion tags: - image-generation model_name: Stable Diffusion python_version: py39 requirements: -- diffusers -- transformers -- torch -- scipy -- accelerate -- pillow +- diffusers==0.24.0 +- transformers==4.36.0 +- torch==2.1.0 +- scipy==1.11.4 +- accelerate==0.25.0 +- pillow==10.1.0 resources: accelerator: A10G cpu: '3' memory: 14Gi use_gpu: true secrets: {} -spec_version: 2.0 +spec_version: "2.0" system_packages: [] diff --git a/stable-diffusion/stable-diffusion/data/model_index.json b/image/stable-diffusion/stable-diffusion/data/model_index.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/model_index.json rename to image/stable-diffusion/stable-diffusion/data/model_index.json diff --git a/stable-diffusion/stable-diffusion/data/scheduler/scheduler_config.json b/image/stable-diffusion/stable-diffusion/data/scheduler/scheduler_config.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/scheduler/scheduler_config.json rename to image/stable-diffusion/stable-diffusion/data/scheduler/scheduler_config.json diff --git a/stable-diffusion/stable-diffusion/data/text_encoder/config.json b/image/stable-diffusion/stable-diffusion/data/text_encoder/config.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/text_encoder/config.json rename to image/stable-diffusion/stable-diffusion/data/text_encoder/config.json diff --git a/stable-diffusion/stable-diffusion/data/tokenizer/merges.txt b/image/stable-diffusion/stable-diffusion/data/tokenizer/merges.txt similarity index 100% rename from stable-diffusion/stable-diffusion/data/tokenizer/merges.txt rename to image/stable-diffusion/stable-diffusion/data/tokenizer/merges.txt diff --git a/stable-diffusion/stable-diffusion/data/tokenizer/special_tokens_map.json b/image/stable-diffusion/stable-diffusion/data/tokenizer/special_tokens_map.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/tokenizer/special_tokens_map.json rename to image/stable-diffusion/stable-diffusion/data/tokenizer/special_tokens_map.json diff --git a/stable-diffusion/stable-diffusion/data/tokenizer/tokenizer_config.json b/image/stable-diffusion/stable-diffusion/data/tokenizer/tokenizer_config.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/tokenizer/tokenizer_config.json rename to image/stable-diffusion/stable-diffusion/data/tokenizer/tokenizer_config.json diff --git a/stable-diffusion/stable-diffusion/data/tokenizer/vocab.json b/image/stable-diffusion/stable-diffusion/data/tokenizer/vocab.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/tokenizer/vocab.json rename to image/stable-diffusion/stable-diffusion/data/tokenizer/vocab.json diff --git a/stable-diffusion/stable-diffusion/data/unet/config.json b/image/stable-diffusion/stable-diffusion/data/unet/config.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/unet/config.json rename to image/stable-diffusion/stable-diffusion/data/unet/config.json diff --git a/stable-diffusion/stable-diffusion/data/vae/config.json b/image/stable-diffusion/stable-diffusion/data/vae/config.json similarity index 100% rename from stable-diffusion/stable-diffusion/data/vae/config.json rename to image/stable-diffusion/stable-diffusion/data/vae/config.json diff --git a/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/__init__.py b/image/stable-diffusion/stable-diffusion/model/__init__.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/__init__.py rename to image/stable-diffusion/stable-diffusion/model/__init__.py diff --git a/stable-diffusion/stable-diffusion/model/model.py b/image/stable-diffusion/stable-diffusion/model/model.py similarity index 100% rename from stable-diffusion/stable-diffusion/model/model.py rename to image/stable-diffusion/stable-diffusion/model/model.py diff --git a/stable-diffusion/stable-diffusion/show.py b/image/stable-diffusion/stable-diffusion/show.py similarity index 100% rename from stable-diffusion/stable-diffusion/show.py rename to image/stable-diffusion/stable-diffusion/show.py diff --git a/image/stable-diffusion/stable-video-diffusion/README.md b/image/stable-diffusion/stable-video-diffusion/README.md new file mode 100644 index 000000000..6eea00f77 --- /dev/null +++ b/image/stable-diffusion/stable-video-diffusion/README.md @@ -0,0 +1,31 @@ +# Stable Video Diffusion + +Stable Video Diffusion can turn any image into a short video. + +| Property | Value | +|----------|-------| +| Task | Image generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "A photo of a cat in a field of sunflowers"}' +``` + +> The response may contain base64-encoded image data. + +## Configuration highlights + +- System packages: `libgl1-mesa-glx, ffmpeg` diff --git a/stable-diffusion/stable-video-diffusion/config.yaml b/image/stable-diffusion/stable-video-diffusion/config.yaml similarity index 91% rename from stable-diffusion/stable-video-diffusion/config.yaml rename to image/stable-diffusion/stable-video-diffusion/config.yaml index 3dc105678..c8c150352 100644 --- a/stable-diffusion/stable-video-diffusion/config.yaml +++ b/image/stable-diffusion/stable-video-diffusion/config.yaml @@ -7,8 +7,11 @@ external_data: url: https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt external_package_dirs: [] model_metadata: + repo_id: "stabilityai/stable-video-diffusion-img2vid" avatar_url: https://cdn.baseten.co/production/static/explore/stability.png cover_image_url: https://cdn.baseten.co/production/static/explore/stable-video-diffusion.png + example_model_input: + image: "" tags: - image-to-video model_name: Stable Video Diffusion diff --git a/mistral/mixtral-8x7b-instruct-vllm/model/__init__.py b/image/stable-diffusion/stable-video-diffusion/model/__init__.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-vllm/model/__init__.py rename to image/stable-diffusion/stable-video-diffusion/model/__init__.py diff --git a/stable-diffusion/stable-video-diffusion/model/helper.py b/image/stable-diffusion/stable-video-diffusion/model/helper.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/helper.py rename to image/stable-diffusion/stable-video-diffusion/model/helper.py diff --git a/stable-diffusion/stable-video-diffusion/model/model.py b/image/stable-diffusion/stable-video-diffusion/model/model.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/model.py rename to image/stable-diffusion/stable-video-diffusion/model/model.py diff --git a/mistral/pixtral-12b/model/__init__.py b/image/stable-diffusion/stable-video-diffusion/model/scripts/__init__.py similarity index 100% rename from mistral/pixtral-12b/model/__init__.py rename to image/stable-diffusion/stable-video-diffusion/model/scripts/__init__.py diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd.yaml b/image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd.yaml similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd.yaml rename to image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd.yaml diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_image_decoder.yaml b/image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_image_decoder.yaml similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_image_decoder.yaml rename to image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_image_decoder.yaml diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt.yaml b/image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt.yaml similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt.yaml rename to image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt.yaml diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt_image_decoder.yaml b/image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt_image_decoder.yaml similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt_image_decoder.yaml rename to image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/configs/svd_xt_image_decoder.yaml diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/sampling/simple_video_sample.py b/image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/simple_video_sample.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/sampling/simple_video_sample.py rename to image/stable-diffusion/stable-video-diffusion/model/scripts/sampling/simple_video_sample.py diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/tests/attention.py b/image/stable-diffusion/stable-video-diffusion/model/scripts/tests/attention.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/tests/attention.py rename to image/stable-diffusion/stable-video-diffusion/model/scripts/tests/attention.py diff --git a/multiprocessing/model/__init__.py b/image/stable-diffusion/stable-video-diffusion/model/scripts/util/__init__.py similarity index 100% rename from multiprocessing/model/__init__.py rename to image/stable-diffusion/stable-video-diffusion/model/scripts/util/__init__.py diff --git a/musicgen-large/model/__init__.py b/image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/__init__.py similarity index 100% rename from musicgen-large/model/__init__.py rename to image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/__init__.py diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/nsfw_and_watermark_dectection.py b/image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/nsfw_and_watermark_dectection.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/util/detection/nsfw_and_watermark_dectection.py rename to image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/nsfw_and_watermark_dectection.py diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/p_head_v1.npz b/image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/p_head_v1.npz similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/util/detection/p_head_v1.npz rename to image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/p_head_v1.npz diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/w_head_v1.npz b/image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/w_head_v1.npz similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/util/detection/w_head_v1.npz rename to image/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/w_head_v1.npz diff --git a/stable-diffusion/stable-video-diffusion/sample_images/cheetah.jpeg b/image/stable-diffusion/stable-video-diffusion/sample_images/cheetah.jpeg similarity index 100% rename from stable-diffusion/stable-video-diffusion/sample_images/cheetah.jpeg rename to image/stable-diffusion/stable-video-diffusion/sample_images/cheetah.jpeg diff --git a/stable-diffusion/stable-video-diffusion/sample_images/racecar.jpeg b/image/stable-diffusion/stable-video-diffusion/sample_images/racecar.jpeg similarity index 100% rename from stable-diffusion/stable-video-diffusion/sample_images/racecar.jpeg rename to image/stable-diffusion/stable-video-diffusion/sample_images/racecar.jpeg diff --git a/infrastructure/README.md b/infrastructure/README.md new file mode 100644 index 000000000..2022564dd --- /dev/null +++ b/infrastructure/README.md @@ -0,0 +1,28 @@ +# Infrastructure Examples + +Examples demonstrating advanced Truss features, custom serving infrastructure, and integrations. These cover topics like custom servers, gRPC, chaining models, caching, and specialized tooling. + +| Directory | Description | +|-----------|-------------| +| [custom-server](custom-server/) | Custom inference servers using SGLang, LMDeploy, and other engines with Dockerfile-based Truss configs | +| [grpc](grpc/) | Serve a model over gRPC instead of HTTP | +| [custom-engine-builder-control](custom-engine-builder-control/) | Custom engine builder with fine-grained control over the build process | +| [chains-examples](chains-examples/) | Truss Chains examples for multi-model pipelines | +| [multiprocessing](multiprocessing/) | Use Python multiprocessing within a Truss model | +| [model-cache](model-cache/) | Cache model weights across deployments for faster cold starts | +| [metrics](metrics/) | Export custom metrics from a Truss model | +| [jsonformatter](jsonformatter/) | Custom JSON formatting for model outputs | +| [ngram-speculator](ngram-speculator/) | N-gram speculative decoding for faster LLM inference | +| [llama-cpp-server](llama-cpp-server/) | Serve models using llama.cpp as the backend | +| [binocular](binocular/) | Binocular LLM-generated text detection | +| [layoutlm-document-qa](layoutlm-document-qa/) | LayoutLM document question answering | +| [autodesk-wala](autodesk-wala/) | Autodesk WALA model integration | +| [paddlepaddle](paddlepaddle/) | PaddlePaddle framework model serving | + +## Deploying + +Each example can be deployed to Baseten with: + +```bash +truss push +``` diff --git a/infrastructure/autodesk-wala/README.md b/infrastructure/autodesk-wala/README.md new file mode 100644 index 000000000..ab2d06db7 --- /dev/null +++ b/infrastructure/autodesk-wala/README.md @@ -0,0 +1,31 @@ +# ADSKAILab/WaLa-SV-1B + +Deploy ADSKAILab/WaLa-SV-1B using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Task | Infrastructure / Custom server | +| Engine | Custom (Truss) | +| GPU | H100_40GB | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"image_b64": ""}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/autodesk-wala/config.yaml b/infrastructure/autodesk-wala/config.yaml similarity index 54% rename from autodesk-wala/config.yaml rename to infrastructure/autodesk-wala/config.yaml index ba3cff85f..d5e1ea751 100644 --- a/autodesk-wala/config.yaml +++ b/infrastructure/autodesk-wala/config.yaml @@ -1,3 +1,7 @@ +description: "Autodesk WaLa SV 1B via custom server" +model_metadata: + repo_id: "ADSKAILab/WaLa-SV-1B" + example_model_input: {"image_b64": "", "output_format": "obj", "seed": 42} model_name: ADSKAILab/WaLa-SV-1B python_version: py311 resources: diff --git a/autodesk-wala/examples/README.md b/infrastructure/autodesk-wala/examples/README.md similarity index 100% rename from autodesk-wala/examples/README.md rename to infrastructure/autodesk-wala/examples/README.md diff --git a/autodesk-wala/examples/depth_maps_4/10.png b/infrastructure/autodesk-wala/examples/depth_maps_4/10.png similarity index 100% rename from autodesk-wala/examples/depth_maps_4/10.png rename to infrastructure/autodesk-wala/examples/depth_maps_4/10.png diff --git a/autodesk-wala/examples/depth_maps_4/26.png b/infrastructure/autodesk-wala/examples/depth_maps_4/26.png similarity index 100% rename from autodesk-wala/examples/depth_maps_4/26.png rename to infrastructure/autodesk-wala/examples/depth_maps_4/26.png diff --git a/autodesk-wala/examples/depth_maps_4/3.png b/infrastructure/autodesk-wala/examples/depth_maps_4/3.png similarity index 100% rename from autodesk-wala/examples/depth_maps_4/3.png rename to infrastructure/autodesk-wala/examples/depth_maps_4/3.png diff --git a/autodesk-wala/examples/depth_maps_4/6.png b/infrastructure/autodesk-wala/examples/depth_maps_4/6.png similarity index 100% rename from autodesk-wala/examples/depth_maps_4/6.png rename to infrastructure/autodesk-wala/examples/depth_maps_4/6.png diff --git a/autodesk-wala/examples/depth_maps_6/10.png b/infrastructure/autodesk-wala/examples/depth_maps_6/10.png similarity index 100% rename from autodesk-wala/examples/depth_maps_6/10.png rename to infrastructure/autodesk-wala/examples/depth_maps_6/10.png diff --git a/autodesk-wala/examples/depth_maps_6/26.png b/infrastructure/autodesk-wala/examples/depth_maps_6/26.png similarity index 100% rename from autodesk-wala/examples/depth_maps_6/26.png rename to infrastructure/autodesk-wala/examples/depth_maps_6/26.png diff --git a/autodesk-wala/examples/depth_maps_6/3.png b/infrastructure/autodesk-wala/examples/depth_maps_6/3.png similarity index 100% rename from autodesk-wala/examples/depth_maps_6/3.png rename to infrastructure/autodesk-wala/examples/depth_maps_6/3.png diff --git a/autodesk-wala/examples/depth_maps_6/49.png b/infrastructure/autodesk-wala/examples/depth_maps_6/49.png similarity index 100% rename from autodesk-wala/examples/depth_maps_6/49.png rename to infrastructure/autodesk-wala/examples/depth_maps_6/49.png diff --git a/autodesk-wala/examples/depth_maps_6/50.png b/infrastructure/autodesk-wala/examples/depth_maps_6/50.png similarity index 100% rename from autodesk-wala/examples/depth_maps_6/50.png rename to infrastructure/autodesk-wala/examples/depth_maps_6/50.png diff --git a/autodesk-wala/examples/depth_maps_6/6.png b/infrastructure/autodesk-wala/examples/depth_maps_6/6.png similarity index 100% rename from autodesk-wala/examples/depth_maps_6/6.png rename to infrastructure/autodesk-wala/examples/depth_maps_6/6.png diff --git a/autodesk-wala/examples/multi_view/003.png b/infrastructure/autodesk-wala/examples/multi_view/003.png similarity index 100% rename from autodesk-wala/examples/multi_view/003.png rename to infrastructure/autodesk-wala/examples/multi_view/003.png diff --git a/autodesk-wala/examples/multi_view/006.png b/infrastructure/autodesk-wala/examples/multi_view/006.png similarity index 100% rename from autodesk-wala/examples/multi_view/006.png rename to infrastructure/autodesk-wala/examples/multi_view/006.png diff --git a/autodesk-wala/examples/multi_view/010.png b/infrastructure/autodesk-wala/examples/multi_view/010.png similarity index 100% rename from autodesk-wala/examples/multi_view/010.png rename to infrastructure/autodesk-wala/examples/multi_view/010.png diff --git a/autodesk-wala/examples/multi_view/026.png b/infrastructure/autodesk-wala/examples/multi_view/026.png similarity index 100% rename from autodesk-wala/examples/multi_view/026.png rename to infrastructure/autodesk-wala/examples/multi_view/026.png diff --git a/autodesk-wala/examples/pointcloud/ring.h5df b/infrastructure/autodesk-wala/examples/pointcloud/ring.h5df similarity index 100% rename from autodesk-wala/examples/pointcloud/ring.h5df rename to infrastructure/autodesk-wala/examples/pointcloud/ring.h5df diff --git a/autodesk-wala/examples/single_depth_map/49.png b/infrastructure/autodesk-wala/examples/single_depth_map/49.png similarity index 100% rename from autodesk-wala/examples/single_depth_map/49.png rename to infrastructure/autodesk-wala/examples/single_depth_map/49.png diff --git a/autodesk-wala/examples/single_view/table.png b/infrastructure/autodesk-wala/examples/single_view/table.png similarity index 100% rename from autodesk-wala/examples/single_view/table.png rename to infrastructure/autodesk-wala/examples/single_view/table.png diff --git a/autodesk-wala/examples/sketch/tree.png b/infrastructure/autodesk-wala/examples/sketch/tree.png similarity index 100% rename from autodesk-wala/examples/sketch/tree.png rename to infrastructure/autodesk-wala/examples/sketch/tree.png diff --git a/autodesk-wala/examples/voxel/horse_16.json b/infrastructure/autodesk-wala/examples/voxel/horse_16.json similarity index 100% rename from autodesk-wala/examples/voxel/horse_16.json rename to infrastructure/autodesk-wala/examples/voxel/horse_16.json diff --git a/autodesk-wala/examples/voxel/horse_32.json b/infrastructure/autodesk-wala/examples/voxel/horse_32.json similarity index 100% rename from autodesk-wala/examples/voxel/horse_32.json rename to infrastructure/autodesk-wala/examples/voxel/horse_32.json diff --git a/musicgen-melody/model/__init__.py b/infrastructure/autodesk-wala/model/__init__.py similarity index 100% rename from musicgen-melody/model/__init__.py rename to infrastructure/autodesk-wala/model/__init__.py diff --git a/autodesk-wala/model/model.py b/infrastructure/autodesk-wala/model/model.py similarity index 100% rename from autodesk-wala/model/model.py rename to infrastructure/autodesk-wala/model/model.py diff --git a/autodesk-wala/output.obj b/infrastructure/autodesk-wala/output.obj similarity index 100% rename from autodesk-wala/output.obj rename to infrastructure/autodesk-wala/output.obj diff --git a/ngram-speculator/truss/model/__init__.py b/infrastructure/autodesk-wala/packages/src/__init__.py similarity index 100% rename from ngram-speculator/truss/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/__init__.py diff --git a/autodesk-wala/packages/src/bpe_simple_vocab_16e6.txt.gz b/infrastructure/autodesk-wala/packages/src/bpe_simple_vocab_16e6.txt.gz similarity index 100% rename from autodesk-wala/packages/src/bpe_simple_vocab_16e6.txt.gz rename to infrastructure/autodesk-wala/packages/src/bpe_simple_vocab_16e6.txt.gz diff --git a/autodesk-wala/packages/src/clip_mod.py b/infrastructure/autodesk-wala/packages/src/clip_mod.py similarity index 100% rename from autodesk-wala/packages/src/clip_mod.py rename to infrastructure/autodesk-wala/packages/src/clip_mod.py diff --git a/autodesk-wala/packages/src/dataset_utils.py b/infrastructure/autodesk-wala/packages/src/dataset_utils.py similarity index 100% rename from autodesk-wala/packages/src/dataset_utils.py rename to infrastructure/autodesk-wala/packages/src/dataset_utils.py diff --git a/nous-capybara/nous-capybara-34b-openai/model/__init__.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/__init__.py similarity index 100% rename from nous-capybara/nous-capybara-34b-openai/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/__init__.py diff --git a/autodesk-wala/packages/src/diffusion_modules/dwt.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/dwt.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/dwt.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/dwt.py diff --git a/autodesk-wala/packages/src/diffusion_modules/dwt_utils.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/dwt_utils.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/dwt_utils.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/dwt_utils.py diff --git a/autodesk-wala/packages/src/diffusion_modules/fp16_util.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/fp16_util.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/fp16_util.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/fp16_util.py diff --git a/autodesk-wala/packages/src/diffusion_modules/gaussian_diffusion.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/gaussian_diffusion.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/gaussian_diffusion.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/gaussian_diffusion.py diff --git a/autodesk-wala/packages/src/diffusion_modules/latent_points.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/latent_points.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/latent_points.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/latent_points.py diff --git a/nous-capybara/nous-capybara-34b/model/__init__.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/__init__ copy.py similarity index 100% rename from nous-capybara/nous-capybara-34b/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/__init__ copy.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/__init__.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/__init__.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/__init__.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/__init__.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/ball_query.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/ball_query.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/ball_query.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/ball_query.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/frustum.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/frustum.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/frustum.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/frustum.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/__init__.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/__init__.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/__init__.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/__init__.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/backend.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/backend.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/backend.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/backend.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/ball_query.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/ball_query.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/ball_query.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/ball_query.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/devoxelization.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/devoxelization.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/devoxelization.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/devoxelization.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/grouping.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/grouping.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/grouping.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/grouping.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/interpolatation.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/interpolatation.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/interpolatation.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/interpolatation.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/loss.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/loss.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/loss.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/loss.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/sampling.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/sampling.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/sampling.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/sampling.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cu b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cu similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cu rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cu diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cuh b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cuh similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cuh rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.cuh diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.hpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.hpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.hpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/ball_query/ball_query.hpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/bindings.cpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/bindings.cpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/bindings.cpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/bindings.cpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/cuda_utils.cuh b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/cuda_utils.cuh similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/cuda_utils.cuh rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/cuda_utils.cuh diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cu b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cu similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cu rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cu diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cuh b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cuh similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cuh rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.cuh diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.hpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.hpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.hpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/grouping/grouping.hpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cu b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cu similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cu rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cu diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cuh b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cuh similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cuh rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.cuh diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.hpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.hpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.hpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/neighbor_interpolate.hpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cu b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cu similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cu rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cu diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cuh b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cuh similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cuh rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.cuh diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.hpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.hpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.hpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/interpolate/trilinear_devox.hpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cu b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cu similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cu rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cu diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cuh b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cuh similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cuh rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.cuh diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.hpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.hpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.hpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/sampling/sampling.hpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/utils.hpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/utils.hpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/utils.hpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/utils.hpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cu b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cu similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cu rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cu diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cuh b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cuh similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cuh rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.cuh diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.hpp b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.hpp similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.hpp rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/src/voxelization/vox.hpp diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/functional/voxelization.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/voxelization.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/functional/voxelization.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/functional/voxelization.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/loss.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/loss.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/loss.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/loss.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/pointnet.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/pointnet.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/pointnet.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/pointnet.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/pvconv.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/pvconv.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/pvconv.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/pvconv.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/se.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/se.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/se.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/se.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/shared_mlp.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/shared_mlp.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/shared_mlp.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/shared_mlp.py diff --git a/autodesk-wala/packages/src/diffusion_modules/modules/voxelization.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/voxelization.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/modules/voxelization.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/modules/voxelization.py diff --git a/autodesk-wala/packages/src/diffusion_modules/network_ae.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/network_ae.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/network_ae.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/network_ae.py diff --git a/autodesk-wala/packages/src/diffusion_modules/nn.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/nn.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/nn.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/nn.py diff --git a/autodesk-wala/packages/src/diffusion_modules/point_voxels.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/point_voxels.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/point_voxels.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/point_voxels.py diff --git a/autodesk-wala/packages/src/diffusion_modules/resample.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/resample.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/resample.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/resample.py diff --git a/autodesk-wala/packages/src/diffusion_modules/sparse_network.py b/infrastructure/autodesk-wala/packages/src/diffusion_modules/sparse_network.py similarity index 100% rename from autodesk-wala/packages/src/diffusion_modules/sparse_network.py rename to infrastructure/autodesk-wala/packages/src/diffusion_modules/sparse_network.py diff --git a/nsql/model/__init__.py b/infrastructure/autodesk-wala/packages/src/experiments/__init__.py similarity index 100% rename from nsql/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/experiments/__init__.py diff --git a/nvidia/parakeet-tdt-0_6b-v2/model/__init__.py b/infrastructure/autodesk-wala/packages/src/experiments/utils/__init__.py similarity index 100% rename from nvidia/parakeet-tdt-0_6b-v2/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/experiments/utils/__init__.py diff --git a/autodesk-wala/packages/src/experiments/utils/wavelet_utils.py b/infrastructure/autodesk-wala/packages/src/experiments/utils/wavelet_utils.py similarity index 100% rename from autodesk-wala/packages/src/experiments/utils/wavelet_utils.py rename to infrastructure/autodesk-wala/packages/src/experiments/utils/wavelet_utils.py diff --git a/phi/phi-3-mini-128k-instruct/model/__init__.py b/infrastructure/autodesk-wala/packages/src/latent_model/__init__.py similarity index 100% rename from phi/phi-3-mini-128k-instruct/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/latent_model/__init__.py diff --git a/autodesk-wala/packages/src/latent_model/abstract_volume_nn.py b/infrastructure/autodesk-wala/packages/src/latent_model/abstract_volume_nn.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/abstract_volume_nn.py rename to infrastructure/autodesk-wala/packages/src/latent_model/abstract_volume_nn.py diff --git a/autodesk-wala/packages/src/latent_model/continous_diffusion_interface.py b/infrastructure/autodesk-wala/packages/src/latent_model/continous_diffusion_interface.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/continous_diffusion_interface.py rename to infrastructure/autodesk-wala/packages/src/latent_model/continous_diffusion_interface.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/dwt.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/dwt.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/dwt.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/dwt.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/dwt_utils.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/dwt_utils.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/dwt_utils.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/dwt_utils.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/fp16_util.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/fp16_util.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/fp16_util.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/fp16_util.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion_wavelet.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion_wavelet.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion_wavelet.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/gaussian_diffusion_wavelet.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/nn.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/nn.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/nn.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/nn.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/resample.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/resample.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/resample.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/resample.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/sparse_network.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/sparse_network.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/sparse_network.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/sparse_network.py diff --git a/autodesk-wala/packages/src/latent_model/diffusion_modules/utils.py b/infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/utils.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/diffusion_modules/utils.py rename to infrastructure/autodesk-wala/packages/src/latent_model/diffusion_modules/utils.py diff --git a/autodesk-wala/packages/src/latent_model/dit_utils.py b/infrastructure/autodesk-wala/packages/src/latent_model/dit_utils.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/dit_utils.py rename to infrastructure/autodesk-wala/packages/src/latent_model/dit_utils.py diff --git a/autodesk-wala/packages/src/latent_model/gaussian_diffusion.py b/infrastructure/autodesk-wala/packages/src/latent_model/gaussian_diffusion.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/gaussian_diffusion.py rename to infrastructure/autodesk-wala/packages/src/latent_model/gaussian_diffusion.py diff --git a/autodesk-wala/packages/src/latent_model/latent_dit_utils.py b/infrastructure/autodesk-wala/packages/src/latent_model/latent_dit_utils.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/latent_dit_utils.py rename to infrastructure/autodesk-wala/packages/src/latent_model/latent_dit_utils.py diff --git a/autodesk-wala/packages/src/latent_model/latent_uvit_utils.py b/infrastructure/autodesk-wala/packages/src/latent_model/latent_uvit_utils.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/latent_uvit_utils.py rename to infrastructure/autodesk-wala/packages/src/latent_model/latent_uvit_utils.py diff --git a/autodesk-wala/packages/src/latent_model/points_network.py b/infrastructure/autodesk-wala/packages/src/latent_model/points_network.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/points_network.py rename to infrastructure/autodesk-wala/packages/src/latent_model/points_network.py diff --git a/autodesk-wala/packages/src/latent_model/quantize.py b/infrastructure/autodesk-wala/packages/src/latent_model/quantize.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/quantize.py rename to infrastructure/autodesk-wala/packages/src/latent_model/quantize.py diff --git a/autodesk-wala/packages/src/latent_model/utils.py b/infrastructure/autodesk-wala/packages/src/latent_model/utils.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/utils.py rename to infrastructure/autodesk-wala/packages/src/latent_model/utils.py diff --git a/autodesk-wala/packages/src/latent_model/voxels_network.py b/infrastructure/autodesk-wala/packages/src/latent_model/voxels_network.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/voxels_network.py rename to infrastructure/autodesk-wala/packages/src/latent_model/voxels_network.py diff --git a/autodesk-wala/packages/src/latent_model/wavelet_utils.py b/infrastructure/autodesk-wala/packages/src/latent_model/wavelet_utils.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/wavelet_utils.py rename to infrastructure/autodesk-wala/packages/src/latent_model/wavelet_utils.py diff --git a/autodesk-wala/packages/src/latent_model/wavelet_vq_model.py b/infrastructure/autodesk-wala/packages/src/latent_model/wavelet_vq_model.py similarity index 100% rename from autodesk-wala/packages/src/latent_model/wavelet_vq_model.py rename to infrastructure/autodesk-wala/packages/src/latent_model/wavelet_vq_model.py diff --git a/autodesk-wala/packages/src/latent_module.py b/infrastructure/autodesk-wala/packages/src/latent_module.py similarity index 100% rename from autodesk-wala/packages/src/latent_module.py rename to infrastructure/autodesk-wala/packages/src/latent_module.py diff --git a/autodesk-wala/packages/src/model_utils.py b/infrastructure/autodesk-wala/packages/src/model_utils.py similarity index 100% rename from autodesk-wala/packages/src/model_utils.py rename to infrastructure/autodesk-wala/packages/src/model_utils.py diff --git a/autodesk-wala/packages/src/mvdream/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/__init__.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/callbacks.py b/infrastructure/autodesk-wala/packages/src/mvdream/callbacks.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/callbacks.py rename to infrastructure/autodesk-wala/packages/src/mvdream/callbacks.py diff --git a/autodesk-wala/packages/src/mvdream/camera_utils.py b/infrastructure/autodesk-wala/packages/src/mvdream/camera_utils.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/camera_utils.py rename to infrastructure/autodesk-wala/packages/src/mvdream/camera_utils.py diff --git a/autodesk-wala/packages/src/mvdream/coco_prompts.py b/infrastructure/autodesk-wala/packages/src/mvdream/coco_prompts.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/coco_prompts.py rename to infrastructure/autodesk-wala/packages/src/mvdream/coco_prompts.py diff --git a/autodesk-wala/packages/src/mvdream/configs/sd-v1.yaml b/infrastructure/autodesk-wala/packages/src/mvdream/configs/sd-v1.yaml similarity index 100% rename from autodesk-wala/packages/src/mvdream/configs/sd-v1.yaml rename to infrastructure/autodesk-wala/packages/src/mvdream/configs/sd-v1.yaml diff --git a/autodesk-wala/packages/src/mvdream/configs/sd-v2-base.yaml b/infrastructure/autodesk-wala/packages/src/mvdream/configs/sd-v2-base.yaml similarity index 100% rename from autodesk-wala/packages/src/mvdream/configs/sd-v2-base.yaml rename to infrastructure/autodesk-wala/packages/src/mvdream/configs/sd-v2-base.yaml diff --git a/autodesk-wala/packages/src/mvdream/constants.py b/infrastructure/autodesk-wala/packages/src/mvdream/constants.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/constants.py rename to infrastructure/autodesk-wala/packages/src/mvdream/constants.py diff --git a/autodesk-wala/packages/src/mvdream/helper.py b/infrastructure/autodesk-wala/packages/src/mvdream/helper.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/helper.py rename to infrastructure/autodesk-wala/packages/src/mvdream/helper.py diff --git a/phi/phi-3-mini-4k-instruct/model/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/__init__.py similarity index 100% rename from phi/phi-3-mini-4k-instruct/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/interface.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/interface.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/interface.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/interface.py diff --git a/phi/phi-3.5-mini/model/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/__init__.py similarity index 100% rename from phi/phi-3.5-mini/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/models/autoencoder.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/autoencoder.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/models/autoencoder.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/autoencoder.py diff --git a/piper-tts/model/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/__init__.py similarity index 100% rename from piper-tts/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py diff --git a/playground-v2-aesthetic/model/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/__init__.py similarity index 100% rename from playground-v2-aesthetic/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/attention.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/attention.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/attention.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/attention.py diff --git a/qwen/qwen-7b-chat/model/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/__init__.py similarity index 100% rename from qwen/qwen-7b-chat/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/openaimodel.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/openaimodel.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/openaimodel.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/openaimodel.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/util.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/util.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/util.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/util.py diff --git a/qwen/qwen-vl/model/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/distributions/__init__.py similarity index 100% rename from qwen/qwen-vl/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/distributions/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/distributions/distributions.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/distributions/distributions.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/distributions/distributions.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/distributions/distributions.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/ema.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/ema.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/ema.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/ema.py diff --git a/sana/sana_1600M/model/__init__.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/encoders/__init__.py similarity index 100% rename from sana/sana_1600M/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/encoders/__init__.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/modules/encoders/modules.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/encoders/modules.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/modules/encoders/modules.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/encoders/modules.py diff --git a/autodesk-wala/packages/src/mvdream/ldm/util.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/util.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/ldm/util.py rename to infrastructure/autodesk-wala/packages/src/mvdream/ldm/util.py diff --git a/autodesk-wala/packages/src/mvdream/model_zoo.py b/infrastructure/autodesk-wala/packages/src/mvdream/model_zoo.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/model_zoo.py rename to infrastructure/autodesk-wala/packages/src/mvdream/model_zoo.py diff --git a/autodesk-wala/packages/src/mvdream/trainer_old.py b/infrastructure/autodesk-wala/packages/src/mvdream/trainer_old.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/trainer_old.py rename to infrastructure/autodesk-wala/packages/src/mvdream/trainer_old.py diff --git a/autodesk-wala/packages/src/mvdream/trainermodule.py b/infrastructure/autodesk-wala/packages/src/mvdream/trainermodule.py similarity index 100% rename from autodesk-wala/packages/src/mvdream/trainermodule.py rename to infrastructure/autodesk-wala/packages/src/mvdream/trainermodule.py diff --git a/autodesk-wala/packages/src/mvdream_module.py b/infrastructure/autodesk-wala/packages/src/mvdream_module.py similarity index 100% rename from autodesk-wala/packages/src/mvdream_module.py rename to infrastructure/autodesk-wala/packages/src/mvdream_module.py diff --git a/autodesk-wala/packages/src/mvdream_utils.py b/infrastructure/autodesk-wala/packages/src/mvdream_utils.py similarity index 100% rename from autodesk-wala/packages/src/mvdream_utils.py rename to infrastructure/autodesk-wala/packages/src/mvdream_utils.py diff --git a/autodesk-wala/packages/src/networks/callbacks.py b/infrastructure/autodesk-wala/packages/src/networks/callbacks.py similarity index 100% rename from autodesk-wala/packages/src/networks/callbacks.py rename to infrastructure/autodesk-wala/packages/src/networks/callbacks.py diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/__init__.py b/infrastructure/autodesk-wala/packages/src/utils/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/__init__.py rename to infrastructure/autodesk-wala/packages/src/utils/__init__.py diff --git a/autodesk-wala/packages/src/utils/visualization.py b/infrastructure/autodesk-wala/packages/src/utils/visualization.py similarity index 100% rename from autodesk-wala/packages/src/utils/visualization.py rename to infrastructure/autodesk-wala/packages/src/utils/visualization.py diff --git a/autodesk-wala/requirements.txt b/infrastructure/autodesk-wala/requirements.txt similarity index 100% rename from autodesk-wala/requirements.txt rename to infrastructure/autodesk-wala/requirements.txt diff --git a/autodesk-wala/test.py b/infrastructure/autodesk-wala/test.py similarity index 100% rename from autodesk-wala/test.py rename to infrastructure/autodesk-wala/test.py diff --git a/autodesk-wala/vendor_wala.sh b/infrastructure/autodesk-wala/vendor_wala.sh similarity index 100% rename from autodesk-wala/vendor_wala.sh rename to infrastructure/autodesk-wala/vendor_wala.sh diff --git a/infrastructure/binocular/README.md b/infrastructure/binocular/README.md new file mode 100644 index 000000000..598d8d94d --- /dev/null +++ b/infrastructure/binocular/README.md @@ -0,0 +1,30 @@ +# Binoculars + +Deploy [tiiuae/falcon-7b](https://huggingface.co/tiiuae/falcon-7b) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [tiiuae/falcon-7b](https://huggingface.co/tiiuae/falcon-7b) | +| Task | Infrastructure / Custom server | +| Engine | Custom (Truss) | +| GPU | A10G:2 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"text": "Your text to check for AI-generated content goes here. The input should be at least 64 tokens long for reliable detection."}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/binocular/config.yaml b/infrastructure/binocular/config.yaml similarity index 63% rename from binocular/config.yaml rename to infrastructure/binocular/config.yaml index 19d2b43df..2d57b3548 100644 --- a/binocular/config.yaml +++ b/infrastructure/binocular/config.yaml @@ -1,3 +1,4 @@ +description: "Binoculars AI-generated text detector using Falcon 7B" environment_variables: {} external_package_dirs: [] model_cache: @@ -12,6 +13,9 @@ model_cache: ignore_patterns: - coreml/* repo_id: tiiuae/falcon-7b-instruct + use_volume: false +model_metadata: + example_model_input: {"text": "The quick brown fox jumps over the lazy dog. This is a sample text to analyze for AI-generated content detection."} model_name: Binoculars python_version: py311 requirements: diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py b/infrastructure/binocular/model/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py rename to infrastructure/binocular/model/__init__.py diff --git a/binocular/model/model.py b/infrastructure/binocular/model/model.py similarity index 100% rename from binocular/model/model.py rename to infrastructure/binocular/model/model.py diff --git a/binocular/packages/config.py b/infrastructure/binocular/packages/config.py similarity index 100% rename from binocular/packages/config.py rename to infrastructure/binocular/packages/config.py diff --git a/chains-examples/docs/audio-transcription/README.md b/infrastructure/chains-examples/docs/audio-transcription/README.md similarity index 100% rename from chains-examples/docs/audio-transcription/README.md rename to infrastructure/chains-examples/docs/audio-transcription/README.md diff --git a/chains-examples/docs/audio-transcription/data_types.py b/infrastructure/chains-examples/docs/audio-transcription/data_types.py similarity index 100% rename from chains-examples/docs/audio-transcription/data_types.py rename to infrastructure/chains-examples/docs/audio-transcription/data_types.py diff --git a/chains-examples/docs/audio-transcription/helpers.py b/infrastructure/chains-examples/docs/audio-transcription/helpers.py similarity index 100% rename from chains-examples/docs/audio-transcription/helpers.py rename to infrastructure/chains-examples/docs/audio-transcription/helpers.py diff --git a/chains-examples/docs/audio-transcription/transcribe.py b/infrastructure/chains-examples/docs/audio-transcription/transcribe.py similarity index 100% rename from chains-examples/docs/audio-transcription/transcribe.py rename to infrastructure/chains-examples/docs/audio-transcription/transcribe.py diff --git a/chains-examples/docs/audio-transcription/whisper_chainlet.py b/infrastructure/chains-examples/docs/audio-transcription/whisper_chainlet.py similarity index 100% rename from chains-examples/docs/audio-transcription/whisper_chainlet.py rename to infrastructure/chains-examples/docs/audio-transcription/whisper_chainlet.py diff --git a/chains-examples/docs/poems/poems.py b/infrastructure/chains-examples/docs/poems/poems.py similarity index 100% rename from chains-examples/docs/poems/poems.py rename to infrastructure/chains-examples/docs/poems/poems.py diff --git a/infrastructure/custom-engine-builder-control/README.md b/infrastructure/custom-engine-builder-control/README.md new file mode 100644 index 000000000..e10dcddf7 --- /dev/null +++ b/infrastructure/custom-engine-builder-control/README.md @@ -0,0 +1,62 @@ +# Briton-suffix-fanout-qwen3-8B + +Deploy [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) | +| Task | Infrastructure / Custom server | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "max_tokens": 100, + "messages": [ + { + "content": "You are a helpful assistant. To each math question, e.g. , respond with all parts the question, '=', and answer.", + "role": "system" + } + ], + "stream": false, + "suffix_messages": [ + [ + { + "role": "system", + "content": "Whats 1+1" + } + ], + [ + { + "role": "system", + "content": "Whats 2+2" + } + ] + ], + "temperature": 0.5, + "chat_template_kwargs": { + "enable_thinking": false + } +}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **32,768** +- Plugin: **use_fp8_context_fmha** diff --git a/custom-engine-builder-control/config.yaml b/infrastructure/custom-engine-builder-control/config.yaml similarity index 95% rename from custom-engine-builder-control/config.yaml rename to infrastructure/custom-engine-builder-control/config.yaml index 0e7459e85..458946fec 100644 --- a/custom-engine-builder-control/config.yaml +++ b/infrastructure/custom-engine-builder-control/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Custom engine builder control example with Qwen3 8B" model_metadata: example_model_input: max_tokens: 100 diff --git a/custom-engine-builder-control/model/model.py b/infrastructure/custom-engine-builder-control/model/model.py similarity index 100% rename from custom-engine-builder-control/model/model.py rename to infrastructure/custom-engine-builder-control/model/model.py diff --git a/infrastructure/custom-server/README.md b/infrastructure/custom-server/README.md new file mode 100644 index 000000000..1346207df --- /dev/null +++ b/infrastructure/custom-server/README.md @@ -0,0 +1,37 @@ +# Custom Server + +Examples of deploying models using `docker_server.start_command` instead of a traditional `model.py`. This is useful when deploying models that are already wrapped in an API (e.g. vLLM, SGLang, LMDeploy) or when using a custom Docker image that handles HTTP requests directly. + +| Example | Engine | Description | +|---------|--------|-------------| +| [llama3-8b-instruct-sglang](llama3-8b-instruct-sglang/) | SGLang | Llama 3 8B Instruct via SGLang server | +| [llama3-70b-instruct-sglang](llama3-70b-instruct-sglang/) | SGLang | Llama 3 70B Instruct via SGLang server | +| [llama3-70b-instruct-lmdeploy](llama3-70b-instruct-lmdeploy/) | LMDeploy | Llama 3 70B Instruct via LMDeploy server | +| [llama3-8b-instruct-lmdeploy](llama3-8b-instruct-lmdeploy/) | LMDeploy | Llama 3 8B Instruct via LMDeploy server | +| [deepseek-v2-5-instruct-sglang](deepseek-v2-5-instruct-sglang/) | SGLang | DeepSeek v2.5 Instruct via SGLang server | +| [pixtral-12b](pixtral-12b/) | vLLM | Pixtral 12B multimodal model | +| [infinity-embedding-server](infinity-embedding-server/) | Infinity | Embedding server using Infinity engine | +| [ultravox-0.4](ultravox-0.4/) | vLLM | Ultravox 0.4 multimodal audio model | +| [ultravox-0.5-8b](ultravox-0.5-8b/) | vLLM | Ultravox 0.5 8B multimodal audio model | +| [ultravox-0.6-70b](ultravox-0.6-70b/) | vLLM | Ultravox 0.6 70B multimodal audio model | +| [voxtral-mini-3b-2507](voxtral-mini-3b-2507/) | vLLM | Voxtral Mini 3B speech model | +| [voxtral-small-24b-2507](voxtral-small-24b-2507/) | vLLM | Voxtral Small 24B speech model | + +## Deploy + +```sh +truss push infrastructure/custom-server/llama3-8b-instruct-sglang +``` + +## How it works + +Instead of writing a `model.py` with `load()` and `predict()` methods, these examples use `docker_server.start_command` in `config.yaml` to launch an existing inference server: + +```yaml +docker_server: + start_command: "python -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct ..." + predict_endpoint: /v1/chat/completions + server_port: 8000 +``` + +This avoids unnecessary overhead when the model already provides its own HTTP endpoint. diff --git a/infrastructure/custom-server/deepseek-v2-5-instruct-sglang/README.md b/infrastructure/custom-server/deepseek-v2-5-instruct-sglang/README.md new file mode 100644 index 000000000..bdc97795e --- /dev/null +++ b/infrastructure/custom-server/deepseek-v2-5-instruct-sglang/README.md @@ -0,0 +1,32 @@ +# DeepSeek V2.5 1210 SGLang + +Deploy [deepseek-ai/DeepSeek-V2.5-1210](https://huggingface.co/deepseek-ai/DeepSeek-V2.5-1210) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-V2.5-1210](https://huggingface.co/deepseek-ai/DeepSeek-V2.5-1210) | +| Task | Infrastructure / Custom server | +| Engine | SGLang | +| GPU | H100:8 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-V2.5-1210", "prompt": "What is the capital of France?", "max_tokens": 256}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.0.post1-cu124` +- Predict concurrency: **32** diff --git a/custom-server/deepseek-v2-5-instruct-sglang/config.yaml b/infrastructure/custom-server/deepseek-v2-5-instruct-sglang/config.yaml similarity index 76% rename from custom-server/deepseek-v2-5-instruct-sglang/config.yaml rename to infrastructure/custom-server/deepseek-v2-5-instruct-sglang/config.yaml index e85af9083..954417dea 100644 --- a/custom-server/deepseek-v2-5-instruct-sglang/config.yaml +++ b/infrastructure/custom-server/deepseek-v2-5-instruct-sglang/config.yaml @@ -1,6 +1,8 @@ +description: "DeepSeek V2.5 via custom SGLang server" base_image: image: lmsysorg/sglang:v0.4.0.post1-cu124 model_metadata: + example_model_input: {"model": "deepseek-ai/DeepSeek-V2.5-1210", "prompt": "What is machine learning?", "max_tokens": 512} repo_id: deepseek-ai/DeepSeek-V2.5-1210 docker_server: start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V2.5-1210 --port 8000 --tp 8 --trust-remote-code" diff --git a/infrastructure/custom-server/infinity-embedding-server/README.md b/infrastructure/custom-server/infinity-embedding-server/README.md new file mode 100644 index 000000000..9df5e84ea --- /dev/null +++ b/infrastructure/custom-server/infinity-embedding-server/README.md @@ -0,0 +1,33 @@ +# infinity-embedding-server + +Deploy [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5) | +| Task | Infrastructure / Custom server | +| Engine | Docker Server | +| GPU | L4 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/embeddings \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "BAAI/bge-small-en-v1.5", "input": ["What is the meaning of life?"]}' +``` + +## Configuration highlights + +- Base image: `python:3.11-slim` +- Predict concurrency: **40** +- Environment variables: `INFINITY_MAX_CLIENT_BATCH_SIZE`, `INFINITY_QUEUE_SIZE`, `DO_NOT_TRACK` diff --git a/custom-server/infinity-embedding-server/config.yaml b/infrastructure/custom-server/infinity-embedding-server/config.yaml similarity index 83% rename from custom-server/infinity-embedding-server/config.yaml rename to infrastructure/custom-server/infinity-embedding-server/config.yaml index afe6c8f3e..38760ae19 100644 --- a/custom-server/infinity-embedding-server/config.yaml +++ b/infrastructure/custom-server/infinity-embedding-server/config.yaml @@ -1,3 +1,4 @@ +description: "BGE Small EN v1.5 via Infinity embedding server" base_image: image: python:3.11-slim docker_server: @@ -11,6 +12,9 @@ build_commands: # optional step to download the weights of the model into the im resources: accelerator: L4 use_gpu: true +model_metadata: + repo_id: "BAAI/bge-small-en-v1.5" + example_model_input: {"input": ["What is deep learning?"], "model": "BAAI/bge-small-en-v1.5"} model_name: infinity-embedding-server requirements: - infinity-emb[all]==0.0.72 diff --git a/infrastructure/custom-server/llama3-70b-instruct-lmdeploy/README.md b/infrastructure/custom-server/llama3-70b-instruct-lmdeploy/README.md new file mode 100644 index 000000000..5aff4c03a --- /dev/null +++ b/infrastructure/custom-server/llama3-70b-instruct-lmdeploy/README.md @@ -0,0 +1,32 @@ +# Llama 3.1 70B Instruct LMDeploy + +Deploy [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) | +| Task | Infrastructure / Custom server | +| Engine | Docker Server | +| GPU | H100:4 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-70B-Instruct", "prompt": "What is the capital of France?", "max_tokens": 256}' +``` + +## Configuration highlights + +- Base image: `openmmlab/lmdeploy:v0.6.4-cu12` +- Predict concurrency: **32** diff --git a/custom-server/llama3-70b-instruct-lmdeploy/config.yaml b/infrastructure/custom-server/llama3-70b-instruct-lmdeploy/config.yaml similarity index 75% rename from custom-server/llama3-70b-instruct-lmdeploy/config.yaml rename to infrastructure/custom-server/llama3-70b-instruct-lmdeploy/config.yaml index b80a57615..925ae964d 100644 --- a/custom-server/llama3-70b-instruct-lmdeploy/config.yaml +++ b/infrastructure/custom-server/llama3-70b-instruct-lmdeploy/config.yaml @@ -1,6 +1,8 @@ +description: "Llama 3.1 70B Instruct via custom LMDeploy server" base_image: image: openmmlab/lmdeploy:v0.6.4-cu12 model_metadata: + example_model_input: {"model": "meta-llama/Llama-3.1-70B-Instruct", "prompt": "What is machine learning?", "max_tokens": 512} repo_id: meta-llama/Llama-3.1-70B-Instruct docker_server: start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python3 -m lmdeploy serve api_server meta-llama/Llama-3.1-70B-Instruct --server-port 8000 --tp 4" diff --git a/infrastructure/custom-server/llama3-70b-instruct-sglang/README.md b/infrastructure/custom-server/llama3-70b-instruct-sglang/README.md new file mode 100644 index 000000000..6d7b41a07 --- /dev/null +++ b/infrastructure/custom-server/llama3-70b-instruct-sglang/README.md @@ -0,0 +1,32 @@ +# Llama 3.1 70B Instruct SGLang + +Deploy [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) | +| Task | Infrastructure / Custom server | +| Engine | SGLang | +| GPU | H100:4 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-70B-Instruct", "prompt": "What is the capital of France?", "max_tokens": 256}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.0.post1-cu124` +- Predict concurrency: **32** diff --git a/custom-server/llama3-70b-instruct-sglang/config.yaml b/infrastructure/custom-server/llama3-70b-instruct-sglang/config.yaml similarity index 75% rename from custom-server/llama3-70b-instruct-sglang/config.yaml rename to infrastructure/custom-server/llama3-70b-instruct-sglang/config.yaml index b7bd233d0..f46bb7b96 100644 --- a/custom-server/llama3-70b-instruct-sglang/config.yaml +++ b/infrastructure/custom-server/llama3-70b-instruct-sglang/config.yaml @@ -1,6 +1,8 @@ +description: "Llama 3.1 70B Instruct via custom SGLang server" base_image: image: lmsysorg/sglang:v0.4.0.post1-cu124 model_metadata: + example_model_input: {"model": "meta-llama/Llama-3.1-70B-Instruct", "prompt": "What is machine learning?", "max_tokens": 512} repo_id: meta-llama/Llama-3.1-70B-Instruct docker_server: start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --port 8000 --tp 4" diff --git a/infrastructure/custom-server/llama3-8b-instruct-lmdeploy/README.md b/infrastructure/custom-server/llama3-8b-instruct-lmdeploy/README.md new file mode 100644 index 000000000..a40d89e4f --- /dev/null +++ b/infrastructure/custom-server/llama3-8b-instruct-lmdeploy/README.md @@ -0,0 +1,32 @@ +# Llama 3.1 8B Instruct LMDeploy + +Deploy [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Task | Infrastructure / Custom server | +| Engine | Docker Server | +| GPU | H100 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-8B-Instruct", "prompt": "What is the capital of France?", "max_tokens": 256}' +``` + +## Configuration highlights + +- Base image: `openmmlab/lmdeploy:v0.6.4-cu12` +- Predict concurrency: **32** diff --git a/custom-server/llama3-8b-instruct-lmdeploy/config.yaml b/infrastructure/custom-server/llama3-8b-instruct-lmdeploy/config.yaml similarity index 74% rename from custom-server/llama3-8b-instruct-lmdeploy/config.yaml rename to infrastructure/custom-server/llama3-8b-instruct-lmdeploy/config.yaml index a1d646eff..057f88a1c 100644 --- a/custom-server/llama3-8b-instruct-lmdeploy/config.yaml +++ b/infrastructure/custom-server/llama3-8b-instruct-lmdeploy/config.yaml @@ -1,6 +1,8 @@ +description: "Llama 3.1 8B Instruct via custom LMDeploy server" base_image: image: openmmlab/lmdeploy:v0.6.4-cu12 model_metadata: + example_model_input: {"model": "meta-llama/Llama-3.1-8B-Instruct", "prompt": "What is machine learning?", "max_tokens": 512} repo_id: meta-llama/Llama-3.1-8B-Instruct docker_server: start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python3 -m lmdeploy serve api_server meta-llama/Llama-3.1-8B-Instruct --server-port 8000" diff --git a/infrastructure/custom-server/llama3-8b-instruct-sglang/README.md b/infrastructure/custom-server/llama3-8b-instruct-sglang/README.md new file mode 100644 index 000000000..9f9511e7a --- /dev/null +++ b/infrastructure/custom-server/llama3-8b-instruct-sglang/README.md @@ -0,0 +1,32 @@ +# Llama 3.1 8B Instruct SGLang + +Deploy [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Task | Infrastructure / Custom server | +| Engine | SGLang | +| GPU | H100 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-8B-Instruct", "prompt": "What is the capital of France?", "max_tokens": 256}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.0.post1-cu124` +- Predict concurrency: **32** diff --git a/custom-server/llama3-8b-instruct-sglang/config.yaml b/infrastructure/custom-server/llama3-8b-instruct-sglang/config.yaml similarity index 75% rename from custom-server/llama3-8b-instruct-sglang/config.yaml rename to infrastructure/custom-server/llama3-8b-instruct-sglang/config.yaml index bb22677c5..6926c2dc4 100644 --- a/custom-server/llama3-8b-instruct-sglang/config.yaml +++ b/infrastructure/custom-server/llama3-8b-instruct-sglang/config.yaml @@ -1,6 +1,8 @@ +description: "Llama 3.1 8B Instruct via custom SGLang server" base_image: image: lmsysorg/sglang:v0.4.0.post1-cu124 model_metadata: + example_model_input: {"model": "meta-llama/Llama-3.1-8B-Instruct", "prompt": "What is machine learning?", "max_tokens": 512} repo_id: meta-llama/Llama-3.1-8B-Instruct docker_server: start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --port 8000" diff --git a/custom-server/llama3_eval.py b/infrastructure/custom-server/llama3_eval.py similarity index 100% rename from custom-server/llama3_eval.py rename to infrastructure/custom-server/llama3_eval.py diff --git a/infrastructure/custom-server/pixtral-12b/README.md b/infrastructure/custom-server/pixtral-12b/README.md new file mode 100644 index 000000000..ee26c5a69 --- /dev/null +++ b/infrastructure/custom-server/pixtral-12b/README.md @@ -0,0 +1,54 @@ +# Pixtral 12B + +Deploy [mistralai/Pixtral-12B-2409](https://huggingface.co/mistralai/Pixtral-12B-2409) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Pixtral-12B-2409](https://huggingface.co/mistralai/Pixtral-12B-2409) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "pixtral", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe this image in one sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://picsum.photos/id/237/200/300" + } + } + ] + } + ], + "stream": false, + "max_tokens": 512, + "temperature": 0.5 +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.7.3` +- Predict concurrency: **16** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/custom-server/pixtral-12b/config.yaml b/infrastructure/custom-server/pixtral-12b/config.yaml similarity index 96% rename from custom-server/pixtral-12b/config.yaml rename to infrastructure/custom-server/pixtral-12b/config.yaml index 22a9f1f5f..4a76d2501 100644 --- a/custom-server/pixtral-12b/config.yaml +++ b/infrastructure/custom-server/pixtral-12b/config.yaml @@ -1,3 +1,4 @@ +description: "Pixtral 12B via custom vLLM server" base_image: image: vllm/vllm-openai:v0.7.3 model_metadata: diff --git a/custom-server/pixtral-12b/data/pixtral12b.jinja b/infrastructure/custom-server/pixtral-12b/data/pixtral12b.jinja similarity index 100% rename from custom-server/pixtral-12b/data/pixtral12b.jinja rename to infrastructure/custom-server/pixtral-12b/data/pixtral12b.jinja diff --git a/infrastructure/custom-server/ultravox-0.4/README.md b/infrastructure/custom-server/ultravox-0.4/README.md new file mode 100644 index 000000000..c6948d86f --- /dev/null +++ b/infrastructure/custom-server/ultravox-0.4/README.md @@ -0,0 +1,50 @@ +# Ultravox v0.4 + +Take in audio and text as input, generating text as usual + +| Property | Value | +|----------|-------| +| Model | [fixie-ai/ultravox-v0.4](https://huggingface.co/fixie-ai/ultravox-v0.4) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100_40GB | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ultravox", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is Lydia like?" + }, + { + "type": "audio_url", + "audio_url": { + "url": "https://baseten-public.s3.us-west-2.amazonaws.com/fred-audio-tests/real.mp3" + } + } + ] + } + ] +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.9.2` +- Predict concurrency: **16** diff --git a/custom-server/ultravox-0.4/config.yaml b/infrastructure/custom-server/ultravox-0.4/config.yaml similarity index 100% rename from custom-server/ultravox-0.4/config.yaml rename to infrastructure/custom-server/ultravox-0.4/config.yaml diff --git a/infrastructure/custom-server/ultravox-0.5-8b/README.md b/infrastructure/custom-server/ultravox-0.5-8b/README.md new file mode 100644 index 000000000..e14f042fd --- /dev/null +++ b/infrastructure/custom-server/ultravox-0.5-8b/README.md @@ -0,0 +1,52 @@ +# Ultravox v0.5 8B + +Take in audio and text as input, generating text as usual + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ultravox", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is Lydia like?" + }, + { + "type": "audio_url", + "audio_url": { + "url": "https://baseten-public.s3.us-west-2.amazonaws.com/fred-audio-tests/real.mp3" + } + } + ] + } + ] +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.9.2` +- Predict concurrency: **16** diff --git a/custom-server/ultravox-0.5-8b/config.yaml b/infrastructure/custom-server/ultravox-0.5-8b/config.yaml similarity index 100% rename from custom-server/ultravox-0.5-8b/config.yaml rename to infrastructure/custom-server/ultravox-0.5-8b/config.yaml diff --git a/infrastructure/custom-server/ultravox-0.6-70b/README.md b/infrastructure/custom-server/ultravox-0.6-70b/README.md new file mode 100644 index 000000000..4a0bbba14 --- /dev/null +++ b/infrastructure/custom-server/ultravox-0.6-70b/README.md @@ -0,0 +1,52 @@ +# Ultravox v0.6 70B + +Take in audio and text as input, generating text as usual + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100:4 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ultravox", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is Lydia like?" + }, + { + "type": "audio_url", + "audio_url": { + "url": "https://baseten-public.s3.us-west-2.amazonaws.com/fred-audio-tests/real.mp3" + } + } + ] + } + ] +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.9.2` +- Predict concurrency: **16** diff --git a/custom-server/ultravox-0.6-70b/config.yaml b/infrastructure/custom-server/ultravox-0.6-70b/config.yaml similarity index 100% rename from custom-server/ultravox-0.6-70b/config.yaml rename to infrastructure/custom-server/ultravox-0.6-70b/config.yaml diff --git a/infrastructure/custom-server/voxtral-mini-3b-2507/README.md b/infrastructure/custom-server/voxtral-mini-3b-2507/README.md new file mode 100644 index 000000000..9da13aa35 --- /dev/null +++ b/infrastructure/custom-server/voxtral-mini-3b-2507/README.md @@ -0,0 +1,46 @@ +# Voxtral Mini 3B 2507 + +Take in audio and text as input, generating text as usual + +| Property | Value | +|----------|-------| +| Model | [mistralai/Voxtral-Mini-3B-2507](https://huggingface.co/mistralai/Voxtral-Mini-3B-2507) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100_40GB | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "voxtral-mini", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is the name of the famous bicycle race in France?" + } + ] + } + ] +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.10.0` +- Predict concurrency: **16** diff --git a/custom-server/voxtral-mini-3b-2507/config.yaml b/infrastructure/custom-server/voxtral-mini-3b-2507/config.yaml similarity index 98% rename from custom-server/voxtral-mini-3b-2507/config.yaml rename to infrastructure/custom-server/voxtral-mini-3b-2507/config.yaml index 199fd5299..2fc4d0a7c 100644 --- a/custom-server/voxtral-mini-3b-2507/config.yaml +++ b/infrastructure/custom-server/voxtral-mini-3b-2507/config.yaml @@ -38,4 +38,4 @@ model_name: Voxtral Mini 3B 2507 secrets: hf_access_token: null requirements: - - vllm[audio] + - vllm[audio]==0.10.0 diff --git a/infrastructure/custom-server/voxtral-small-24b-2507/README.md b/infrastructure/custom-server/voxtral-small-24b-2507/README.md new file mode 100644 index 000000000..27cf0c677 --- /dev/null +++ b/infrastructure/custom-server/voxtral-small-24b-2507/README.md @@ -0,0 +1,46 @@ +# Voxtral Small 24B 2507 + +Take in audio and text as input, generating text as usual + +| Property | Value | +|----------|-------| +| Model | [mistralai/Voxtral-Small-24B-2507](https://huggingface.co/mistralai/Voxtral-Small-24B-2507) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "voxtral-small", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is the name of the famous bicycle race in France?" + } + ] + } + ] +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.10.0` +- Predict concurrency: **16** diff --git a/custom-server/voxtral-small-24b-2507/config.yaml b/infrastructure/custom-server/voxtral-small-24b-2507/config.yaml similarity index 98% rename from custom-server/voxtral-small-24b-2507/config.yaml rename to infrastructure/custom-server/voxtral-small-24b-2507/config.yaml index f85fbef5c..521681e74 100644 --- a/custom-server/voxtral-small-24b-2507/config.yaml +++ b/infrastructure/custom-server/voxtral-small-24b-2507/config.yaml @@ -38,4 +38,4 @@ model_name: Voxtral Small 24B 2507 secrets: hf_access_token: null requirements: - - vllm[audio] + - vllm[audio]==0.10.0 diff --git a/grpc/Dockerfile b/infrastructure/grpc/Dockerfile similarity index 100% rename from grpc/Dockerfile rename to infrastructure/grpc/Dockerfile diff --git a/infrastructure/grpc/README.md b/infrastructure/grpc/README.md new file mode 100644 index 000000000..e46928ade --- /dev/null +++ b/infrastructure/grpc/README.md @@ -0,0 +1,49 @@ +# gRPC Model Example + +Deploy a model using gRPC transport on Baseten. This example demonstrates how to serve a gRPC service using a custom Docker server with Truss. + +| Property | Value | +|----------|-------| +| Task | Infrastructure / gRPC transport | +| Engine | Docker Server | +| GPU | A10G | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model uses **gRPC transport**, not HTTP/REST. Use the included `client.py` as a reference: + +```python +import grpc +import example_pb2 +import example_pb2_grpc + +channel = grpc.insecure_channel( + "model-.grpc.api.baseten.co:80", +) + +stub = example_pb2_grpc.GreeterStub(channel) + +request = example_pb2.HelloRequest(name="World") + +metadata = [ + ("baseten-authorization", "Api-Key YOUR_BASETEN_API_KEY"), + ("baseten-model-id", ""), +] + +response = stub.SayHello(request, metadata=metadata) +print(response.message) +``` + +The proto definition (`example.proto`) defines a simple `Greeter` service with a `SayHello` RPC method. + +## Configuration highlights + +- Transport: **gRPC** (not HTTP/REST -- curl will not work) +- Base image: `your/repository:tag` +- Docker server with custom `model.py` entrypoint diff --git a/grpc/client.py b/infrastructure/grpc/client.py similarity index 100% rename from grpc/client.py rename to infrastructure/grpc/client.py diff --git a/grpc/config.yaml b/infrastructure/grpc/config.yaml similarity index 75% rename from grpc/config.yaml rename to infrastructure/grpc/config.yaml index b8802073a..101368971 100644 --- a/grpc/config.yaml +++ b/infrastructure/grpc/config.yaml @@ -1,3 +1,6 @@ +description: "Example model using gRPC transport" +model_metadata: + example_model_input: {"name": "World"} model_name: "gRPC Model Example" base_image: image: your/repository:tag diff --git a/grpc/example.proto b/infrastructure/grpc/example.proto similarity index 100% rename from grpc/example.proto rename to infrastructure/grpc/example.proto diff --git a/grpc/example_pb2.py b/infrastructure/grpc/example_pb2.py similarity index 100% rename from grpc/example_pb2.py rename to infrastructure/grpc/example_pb2.py diff --git a/grpc/example_pb2_grpc.py b/infrastructure/grpc/example_pb2_grpc.py similarity index 100% rename from grpc/example_pb2_grpc.py rename to infrastructure/grpc/example_pb2_grpc.py diff --git a/grpc/model.py b/infrastructure/grpc/model.py similarity index 100% rename from grpc/model.py rename to infrastructure/grpc/model.py diff --git a/grpc/requirements.txt b/infrastructure/grpc/requirements.txt similarity index 100% rename from grpc/requirements.txt rename to infrastructure/grpc/requirements.txt diff --git a/infrastructure/jsonformatter/README.md b/infrastructure/jsonformatter/README.md new file mode 100644 index 000000000..09d8b51b9 --- /dev/null +++ b/infrastructure/jsonformatter/README.md @@ -0,0 +1,29 @@ +# JsonFormatter + +Deploy JsonFormatter using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Task | Infrastructure / Custom server | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "Generate a person'\''s name and age"}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/jsonformatter/config.yaml b/infrastructure/jsonformatter/config.yaml similarity index 50% rename from jsonformatter/config.yaml rename to infrastructure/jsonformatter/config.yaml index 152d89d42..63216e83f 100644 --- a/jsonformatter/config.yaml +++ b/infrastructure/jsonformatter/config.yaml @@ -1,13 +1,16 @@ +description: "JSON formatting utility model example" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "databricks/dolly-v2-3b" + example_model_input: {"prompt": "Generate a person's information"} llm_model: databricks/dolly-v2-3b model_name: JsonFormatter python_version: py311 requirements: -- jsonformer -- transformers -- accelerate +- jsonformer==0.12.0 +- transformers==4.36.0 +- accelerate==0.25.0 resources: accelerator: A10G secrets: {} diff --git a/jsonformatter/data/schema.json b/infrastructure/jsonformatter/data/schema.json similarity index 100% rename from jsonformatter/data/schema.json rename to infrastructure/jsonformatter/data/schema.json diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py b/infrastructure/jsonformatter/model/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py rename to infrastructure/jsonformatter/model/__init__.py diff --git a/jsonformatter/model/model.py b/infrastructure/jsonformatter/model/model.py similarity index 100% rename from jsonformatter/model/model.py rename to infrastructure/jsonformatter/model/model.py diff --git a/infrastructure/layoutlm-document-qa/README.md b/infrastructure/layoutlm-document-qa/README.md new file mode 100644 index 000000000..92e913164 --- /dev/null +++ b/infrastructure/layoutlm-document-qa/README.md @@ -0,0 +1,32 @@ +# LayoutLM Document QA + +Extract information from images of invoices + +| Property | Value | +|----------|-------| +| Task | Infrastructure / Custom server | +| Engine | Custom (Truss) | +| GPU | CPU | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "What is the invoice number?", + "url": "https://templates.invoicehome.com/invoice-template-us-neat-750px.png" +}' +``` + +## Configuration highlights + +- System packages: `tesseract-ocr` diff --git a/layoutlm-document-qa/config.yaml b/infrastructure/layoutlm-document-qa/config.yaml similarity index 94% rename from layoutlm-document-qa/config.yaml rename to infrastructure/layoutlm-document-qa/config.yaml index d3e151e41..c058010ce 100644 --- a/layoutlm-document-qa/config.yaml +++ b/infrastructure/layoutlm-document-qa/config.yaml @@ -2,6 +2,7 @@ description: Extract information from images of invoices environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "impira/layoutlm-document-qa" avatar_url: https://cdn.baseten.co/production/static/explore/impira-logo.png cover_image_url: https://cdn.baseten.co/production/static/explore/document-qa.png example_model_input: diff --git a/sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py b/infrastructure/layoutlm-document-qa/model/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py rename to infrastructure/layoutlm-document-qa/model/__init__.py diff --git a/layoutlm-document-qa/model/model.py b/infrastructure/layoutlm-document-qa/model/model.py similarity index 100% rename from layoutlm-document-qa/model/model.py rename to infrastructure/layoutlm-document-qa/model/model.py diff --git a/infrastructure/llama-cpp-server/README.md b/infrastructure/llama-cpp-server/README.md new file mode 100644 index 000000000..990c81ca8 --- /dev/null +++ b/infrastructure/llama-cpp-server/README.md @@ -0,0 +1,56 @@ +# llama cpp gemma 3 27b it qat q4_0 + +Deploy [google/gemma-3-27b-it-qat-q4_0-gguf](https://huggingface.co/google/gemma-3-27b-it-qat-q4_0-gguf) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [google/gemma-3-27b-it-qat-q4_0-gguf](https://huggingface.co/google/gemma-3-27b-it-qat-q4_0-gguf) | +| Task | Infrastructure / Custom server | +| Engine | Docker Server | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemma", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe this image in one sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://picsum.photos/id/237/200/300" + } + } + ] + } + ], + "stream": true, + "max_tokens": 512, + "temperature": 0.5 +}' +``` + +## Configuration highlights + +- Base image: `alphatozeta/llama-cpp-server:0.4` +- Predict concurrency: **8** +- Streaming: **enabled** diff --git a/llama-cpp-server/config.yaml b/infrastructure/llama-cpp-server/config.yaml similarity index 97% rename from llama-cpp-server/config.yaml rename to infrastructure/llama-cpp-server/config.yaml index 64ebb7a85..9f10e096f 100644 --- a/llama-cpp-server/config.yaml +++ b/infrastructure/llama-cpp-server/config.yaml @@ -1,3 +1,4 @@ +description: "Gemma 3 27B via llama.cpp server" base_image: image: alphatozeta/llama-cpp-server:0.4 build_commands: diff --git a/llama-cpp-server/cuda.Dockerfile b/infrastructure/llama-cpp-server/cuda.Dockerfile similarity index 100% rename from llama-cpp-server/cuda.Dockerfile rename to infrastructure/llama-cpp-server/cuda.Dockerfile diff --git a/llama-cpp-server/llama_server_help b/infrastructure/llama-cpp-server/llama_server_help similarity index 100% rename from llama-cpp-server/llama_server_help rename to infrastructure/llama-cpp-server/llama_server_help diff --git a/metrics/datadog/Dockerfile b/infrastructure/metrics/datadog/Dockerfile similarity index 100% rename from metrics/datadog/Dockerfile rename to infrastructure/metrics/datadog/Dockerfile diff --git a/infrastructure/metrics/datadog/README.md b/infrastructure/metrics/datadog/README.md new file mode 100644 index 000000000..65d27137e --- /dev/null +++ b/infrastructure/metrics/datadog/README.md @@ -0,0 +1,49 @@ +# truss_fastapi_datadog + +Deploy [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100:1 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What does Tongyi Qianwen mean?" + } + ], + "stream": false, + "model": "qwen30b", + "max_tokens": 512, + "temperature": 0.7 +}' +``` + +## Configuration highlights + +- Base image: `chriswirick/truss_fastapi_datadog_vllm:v0.11.0h` +- Predict concurrency: **32** +- Environment variables: `DD_SITE`, `DD_HOSTNAME`, `DD_SERVICE`, `DD_ENV`, `DD_RUN_PATH`, `DD_AUTH_TOKEN_FILE_PATH`, `DD_INVENTORIES_CHECKS_ENABLED`, `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT`, `DD_CLOUD_PROVIDER_METADATA`, `VLLM_LOGGING_LEVEL` diff --git a/metrics/datadog/config.yaml b/infrastructure/metrics/datadog/config.yaml similarity index 95% rename from metrics/datadog/config.yaml rename to infrastructure/metrics/datadog/config.yaml index f74727ab9..5b449b343 100644 --- a/metrics/datadog/config.yaml +++ b/infrastructure/metrics/datadog/config.yaml @@ -1,3 +1,4 @@ +description: "Datadog metrics integration example with Qwen3 30B" base_image: image: chriswirick/truss_fastapi_datadog_vllm:v0.11.0h docker_server: diff --git a/metrics/datadog/datadog-vllm-guide.md b/infrastructure/metrics/datadog/datadog-vllm-guide.md similarity index 100% rename from metrics/datadog/datadog-vllm-guide.md rename to infrastructure/metrics/datadog/datadog-vllm-guide.md diff --git a/metrics/datadog/vllm_conf.yaml b/infrastructure/metrics/datadog/vllm_conf.yaml similarity index 100% rename from metrics/datadog/vllm_conf.yaml rename to infrastructure/metrics/datadog/vllm_conf.yaml diff --git a/infrastructure/model-cache/README.md b/infrastructure/model-cache/README.md new file mode 100644 index 000000000..79fb3d7c8 --- /dev/null +++ b/infrastructure/model-cache/README.md @@ -0,0 +1,36 @@ +# Model Cache + +Demonstrates how to use Truss `model_cache` to cache model weights across deployments for faster cold starts. This example caches Stable Diffusion XL weights using volume-mounted HuggingFace repos. + +| Property | Value | +|----------|-------| +| Task | Infrastructure / Model caching | +| Engine | Custom (Truss) | +| GPU | CPU | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"text": "hello"}' +``` + +The model returns the input data along with the size of the cached model weights file. + +## Configuration highlights + +- Model cache: **volume-mounted** for fast cold starts +- Two cached HuggingFace repos: `madebyollin/sdxl-vae-fp16-fix` and `stabilityai/stable-diffusion-xl-base-1.0` +- Uses `revision` pinning and `allow_patterns` for selective downloads +- `runtime_secret_name` for authenticated HF access at runtime diff --git a/model_cache/config.yaml b/infrastructure/model-cache/config.yaml similarity index 84% rename from model_cache/config.yaml rename to infrastructure/model-cache/config.yaml index 5129c2096..e8af0f419 100644 --- a/model_cache/config.yaml +++ b/infrastructure/model-cache/config.yaml @@ -1,6 +1,9 @@ +description: "Example of model weight caching for faster cold starts" +model_metadata: + example_model_input: {"prompt": "Hello, world!"} model_name: Hello Model Cache Qwen python_version: py311 -requirements: ["torch"] +requirements: ["torch==2.5.1"] resources: accelerator: null cpu: "1" diff --git a/model_cache/model/model.py b/infrastructure/model-cache/model/model.py similarity index 100% rename from model_cache/model/model.py rename to infrastructure/model-cache/model/model.py diff --git a/infrastructure/multiprocessing/README.md b/infrastructure/multiprocessing/README.md new file mode 100644 index 000000000..03972f245 --- /dev/null +++ b/infrastructure/multiprocessing/README.md @@ -0,0 +1,29 @@ +# Model with multiprocessing pre/post-process + +Deploy Model with multiprocessing pre/post-process using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Task | Infrastructure / Custom server | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"n": 100}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/multiprocessing/config.yaml b/infrastructure/multiprocessing/config.yaml similarity index 65% rename from multiprocessing/config.yaml rename to infrastructure/multiprocessing/config.yaml index d8652b4e3..1d9d12bcb 100644 --- a/multiprocessing/config.yaml +++ b/infrastructure/multiprocessing/config.yaml @@ -1,9 +1,12 @@ +description: "Example of multiprocessing for pre/post-processing" environment_variables: {} external_package_dirs: [] +model_metadata: + example_model_input: {"n": 100} model_name: Model with multiprocessing pre/post-process python_version: py310 requirements: -- torch +- torch==2.1.0 resources: accelerator: A10G cpu: '8' diff --git a/sana/sana_1600M/packages/Sana/diffusion/utils/__init__.py b/infrastructure/multiprocessing/model/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from sana/sana_1600M/packages/Sana/diffusion/utils/__init__.py rename to infrastructure/multiprocessing/model/__init__.py diff --git a/multiprocessing/model/model.py b/infrastructure/multiprocessing/model/model.py similarity index 100% rename from multiprocessing/model/model.py rename to infrastructure/multiprocessing/model/model.py diff --git a/multiprocessing/model/test.py b/infrastructure/multiprocessing/model/test.py similarity index 100% rename from multiprocessing/model/test.py rename to infrastructure/multiprocessing/model/test.py diff --git a/infrastructure/ngram-speculator/truss/README.md b/infrastructure/ngram-speculator/truss/README.md new file mode 100644 index 000000000..07e174c11 --- /dev/null +++ b/infrastructure/ngram-speculator/truss/README.md @@ -0,0 +1,29 @@ +# ngram-speculator + +Deploy ngram-speculator using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Task | Infrastructure / Custom server | +| Engine | Custom (Truss) | +| GPU | H100 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"messages": [{"role": "user", "content": "What is the capital of France?"}], "max_tokens": 256}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/ngram-speculator/truss/config.yaml b/infrastructure/ngram-speculator/truss/config.yaml similarity index 50% rename from ngram-speculator/truss/config.yaml rename to infrastructure/ngram-speculator/truss/config.yaml index ff9ea5ff7..bcb3bf8cb 100644 --- a/ngram-speculator/truss/config.yaml +++ b/infrastructure/ngram-speculator/truss/config.yaml @@ -1,7 +1,10 @@ +description: "N-gram speculative decoding example via Truss" build_commands: [] environment_variables: {} external_package_dirs: [] -model_metadata: {} +model_metadata: + repo_id: "NousResearch/Meta-Llama-3.1-8B-Instruct" + example_model_input: {"messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512} model_name: ngram-speculator python_version: py310 requirements: diff --git a/sana/sana_1600M/packages/Sana/tools/__init__.py b/infrastructure/ngram-speculator/truss/model/__init__.py similarity index 100% rename from sana/sana_1600M/packages/Sana/tools/__init__.py rename to infrastructure/ngram-speculator/truss/model/__init__.py diff --git a/ngram-speculator/truss/model/model.py b/infrastructure/ngram-speculator/truss/model/model.py similarity index 100% rename from ngram-speculator/truss/model/model.py rename to infrastructure/ngram-speculator/truss/model/model.py diff --git a/infrastructure/ngram-speculator/trussless/README.md b/infrastructure/ngram-speculator/trussless/README.md new file mode 100644 index 000000000..1b48220b8 --- /dev/null +++ b/infrastructure/ngram-speculator/trussless/README.md @@ -0,0 +1,47 @@ +# ngram-speculator + +Deploy [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "llama", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What do llamas dream of?" + } + ] + } + ], + "stream": false, + "max_tokens": 512 +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:nightly` +- Predict concurrency: **16** diff --git a/ngram-speculator/trussless/config.yaml b/infrastructure/ngram-speculator/trussless/config.yaml similarity index 92% rename from ngram-speculator/trussless/config.yaml rename to infrastructure/ngram-speculator/trussless/config.yaml index c755bfa58..693ab4386 100644 --- a/ngram-speculator/trussless/config.yaml +++ b/infrastructure/ngram-speculator/trussless/config.yaml @@ -1,3 +1,4 @@ +description: "N-gram speculative decoding example via custom server" base_image: image: vllm/vllm-openai:nightly model_metadata: diff --git a/infrastructure/paddlepaddle/paddleocr-vl/README.md b/infrastructure/paddlepaddle/paddleocr-vl/README.md new file mode 100644 index 000000000..d6cea6a1d --- /dev/null +++ b/infrastructure/paddlepaddle/paddleocr-vl/README.md @@ -0,0 +1,52 @@ +# PaddleOCR-VL + +Deploy [PaddlePaddle/PaddleOCR-VL](https://huggingface.co/PaddlePaddle/PaddleOCR-VL) using a custom server configuration on Baseten. + +| Property | Value | +|----------|-------| +| Model | [PaddlePaddle/PaddleOCR-VL](https://huggingface.co/PaddlePaddle/PaddleOCR-VL) | +| Task | Infrastructure / Custom server | +| Engine | vLLM | +| GPU | H100_40GB | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png" + } + }, + { + "type": "text", + "text": "OCR:" + } + ] + } + ], + "model": "PaddlePaddle/PaddleOCR-VL", + "max_tokens": 4096, + "temperature": 0.0 +}' +``` + +## Configuration highlights + +- Base image: `public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:0bf29fadf5f8b28817fbccb037fb70adaef3f7f1` +- Predict concurrency: **128** diff --git a/paddlepaddle/PaddleOCR_VL/config.yaml b/infrastructure/paddlepaddle/paddleocr-vl/config.yaml similarity index 95% rename from paddlepaddle/PaddleOCR_VL/config.yaml rename to infrastructure/paddlepaddle/paddleocr-vl/config.yaml index 37a105a2c..e252cbcac 100644 --- a/paddlepaddle/PaddleOCR_VL/config.yaml +++ b/infrastructure/paddlepaddle/paddleocr-vl/config.yaml @@ -1,3 +1,4 @@ +description: "PaddleOCR VL for optical character recognition" base_image: image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:0bf29fadf5f8b28817fbccb037fb70adaef3f7f1 # build_commands: diff --git a/ip-adapter/README.md b/ip-adapter/README.md deleted file mode 100644 index 38373cfda..000000000 --- a/ip-adapter/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# IP Adapter - -This is a [Truss](https://truss.baseten.co/) for [IP Adapter](https://github.com/tencent-ailab/IP-Adapter). IP Adapter can create variations of a given input image based on a prompt, while retaining the aesthetic of the origina image. - - -## Deploying IP Adapter - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd ip_adapter -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `ip_adapter` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking IP Adapter - -IP Adapter takes in two inputs, an optional `prompt` and a base64 encoded `image`. The output is a JSON blob with a single key, `result` with another base64 encoded image. - - -```sh -truss predict -d '{"image": "data:image/png;base64,iVBORw0KGgoA..."}' -``` - -You can also invoke your model via a REST API -``` -curl -X POST https://app.baseten.co/model_versions//predict \ - -H "Content-Type: application/json" \ - -d '{ - "image": "data:image/png;base64,iVBORw0KGgoA..." - }' -``` diff --git a/kokoro/README.md b/kokoro/README.md deleted file mode 100644 index b7cf9d7fa..000000000 --- a/kokoro/README.md +++ /dev/null @@ -1,13 +0,0 @@ -Kokoro is a frontier TTS model for its size of 82 million parameters (text in/audio out). -API: -```bash -request: -{"text": "Hello", "voice": "af", "speed": 1.0} - -text: str = defaults to "Hi, I'm kokoro" -voice: str = defaults to "af", available options: "af", "af_bella", "af_sarah", "am_adam", "am_michael", "bf_emma", "bf_isabella", "bm_george", "bm_lewis", "af_nicole", "af_sky" -speed: float = defaults to 1.0. The speed of the audio generated - -reponse: -{"base64": "base64 encoded bytestring"} -``` diff --git a/layoutlm-document-qa/README.md b/layoutlm-document-qa/README.md deleted file mode 100644 index 7f6107773..000000000 --- a/layoutlm-document-qa/README.md +++ /dev/null @@ -1,76 +0,0 @@ - # LayoutLM Document QA Truss - -This repository packages [LayoutLM Document QA](https://huggingface.co/impira/layoutlm-document-qa) as a [Truss](https://truss.baseten.co). - -This multimodal model takes an image of an invoice (PNG or JPEG) and extracts information from it in response to natural language prompts. - -## Deploying LayoutLM Document QA - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd layoutlm-document-qa-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `layoutlm-document-qa-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking LayoutLM Document QA - -LayoutLM takes a dictionary with: - -* `url`: The URL of a PNG or JPEG of an invoice -* `prompt`: The question to ask of the invoice - -Example invocation: - -```sh -truss predict -d '{"url": "https://templates.invoicehome.com/invoice-template-us-neat-750px.png", "prompt": "What is the invoice number?"}' -``` - -Expected response: - -```python -[{'answer': '9.06', 'end': 73, 'score': 0.9910207986831665, 'start': 73}] -``` - -You can also invoke your model via a REST API - -``` -curl -X POST "https://app.baseten.co/models/YOUR_MODEL_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "url": "https://templates.invoicehome.com/invoice-template-us-neat-750px.png", - "prompt": "What is the invoice number?" - }' -``` - -## Hardware notes - -We found this model runs reasonably fast with 4 vCPUs and 16 GiB of RAM, no GPU needed. Invocation times are usually <10 seconds for the first prompt on an image (to account for image download times) then <3 seconds thereafter. - -Default config: - -```yaml -... -resources: - cpu: "4" - memory: 16Gi - use_gpu: false - accelerator: null -... -``` diff --git a/llama-cpp-server/README.md b/llama-cpp-server/README.md deleted file mode 100644 index 01cdb580b..000000000 --- a/llama-cpp-server/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Llama.cpp Baseten model server - -Deploying llama.cpp requires a llama.cpp image with python installed and a config.yaml to deploy it. A sample config.yaml is provided in this repository. This sample deploys llama.cpp with Gemma 3 27B Instruct int4 QAT with the 1B model as a draft model. - -The following are instructions in case you need to build a image from source. We enable some non-default flags to optimize for performance, so please use the provided Dockerfile. - -### Building the docker image from source - -#### Prerequisites - -- Docker -- NVIDIA Docker runtime -- CUDA-capable GPU - -#### Building the docker image - -To build the docker image, use the following command: - -```bash -git clone https://github.com/ggml-org/llama.cpp.git -cp cuda.Dockerfile llama.cpp/.devops/cuda.Dockerfile -cd llama.cpp -docker build -t local/llama.cpp:server-cuda --target server -f .devops/cuda.Dockerfile . -``` - -You can then push this image to a container registry of your choice and then replace the base_image in the config.yaml diff --git a/llama/engine-llama-3-1-70b-instruct/README.md b/llama/engine-llama-3-1-70b-instruct/README.md deleted file mode 100644 index 312373439..000000000 --- a/llama/engine-llama-3-1-70b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Llama 3.1 70B Instruct - -This deployment of Llama 3.1 70B Instruct uses the TensorRT-LLM Engine Builder. - -For details, see: https://docs.baseten.co/performance/examples/llama-trt diff --git a/llama/engine-llama-3-1-8b-instruct/README.md b/llama/engine-llama-3-1-8b-instruct/README.md deleted file mode 100644 index 7f8d872b3..000000000 --- a/llama/engine-llama-3-1-8b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Llama 3.1 8B Instruct - -This deployment of Llama 3.1 8B Instruct uses the TensorRT-LLM Engine Builder. - -For details, see: https://docs.baseten.co/performance/examples/llama-trt diff --git a/llama/engine-llama-3-3-70b-instruct/README.md b/llama/engine-llama-3-3-70b-instruct/README.md deleted file mode 100644 index cd90bb5f1..000000000 --- a/llama/engine-llama-3-3-70b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Llama 3.3 70B Instruct - -This deployment of Llama 3.3 70B Instruct uses the TensorRT-LLM Engine Builder. - -For details, see: https://docs.baseten.co/performance/examples/llama-trt diff --git a/llama/engine-llama-3.1-405b-instruct/README.md b/llama/engine-llama-3.1-405b-instruct/README.md deleted file mode 100644 index 51e71f95a..000000000 --- a/llama/engine-llama-3.1-405b-instruct/README.md +++ /dev/null @@ -1,179 +0,0 @@ -# TensorRT-LLM Briton with meta-llama/Llama-3.1-405B - -This is a Deployment for TensorRT-LLM Briton with meta-llama/Llama-3.1-405B. Briton is Baseten's solution for production-grade deployments via TensorRT-LLM for Causal Language Models models. (e.g. LLama, Qwen, Mistral) - -With Briton you get the following benefits by default: -- *Lowest-latency* latency, beating frameworks such as vllm -- *Highest-throughput* inference, automatically using XQA kernels, paged kv caching and inflight batching. -- *distributed inference* run large models (such as LLama-405B) tensor-parallel -- *json-schema based structured output for any model* -- *chunked prefilling* for long generation tasks - -Optionally, you can also enable: -- *speculative decoding* using an external draft model or self-speculative decoding -- *fp8 quantization* deployments on H100, H200 and L4 GPUs - - -# Examples: -This deployment is specifically designed for the Hugging Face model [meta-llama/Llama-3.1-405B](https://huggingface.co/meta-llama/Llama-3.1-405B). -Suitable models can be identified by the `ForCausalLM` suffix in the model name. Currently we support e.g. LLama, Qwen, Mistral models. - -meta-llama/Llama-3.1-405B is a text-generation model, used to generate text given a prompt. \nIt is frequently used in chatbots, text completion, structured output and more. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model Briton-meta-llama-llama-3.1-405b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### OpenAI compatible inference -This solution is OpenAI compatible, which means you can use the OpenAI client library to interact with the model. - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -# Default completion -response_completion = client.completions.create( - model="not_required", - prompt="Q: Tell me everything about Baseten.co! A:", - temperature=0.3, - max_tokens=100, -) - -# Chat completion -response_chat = client.chat.completions.create( - model="", - messages=[ - {"role": "user", "content": "Tell me everything about Baseten.co!"} - ], - temperature=0.3, - max_tokens=100, -) - -# Structured output -from pydantic import BaseModel - -class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - -completion = client.beta.chat.completions.parse( - model="not_required", - messages=[ - {"role": "system", "content": "Extract the event information."}, - {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, - ], - response_format=CalendarEvent, -) - -event = completion.choices[0].message.parsed - -# If you model supports tool-calling, you can use the following example: -tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. BogotΓ‘, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } -}] - -completion = client.chat.completions.create( - model="not_required", - messages=[{"role": "user", "content": "What is the weather like in Paris today?"}], - tools=tools -) - -print(completion.choices[0].message.tool_calls) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8_kv`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. -Note: [This is a gated/private model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Do not set the actual value of key in the config.yaml. `hf_access_token: null` is fine - the true value will be fetched from the secret store. -```yaml -build_commands: [] -environment_variables: - ENABLE_EXECUTOR_API: 1 -external_package_dirs: [] -model_metadata: - example_model_input: - max_tokens: 512 - messages: - - content: Tell me everything you know about optimized inference. - role: user - stream: true - temperature: 0.5 - tags: - - openai-compatible -model_name: Briton-meta-llama-llama-3.1-405b-fp8-truss-example -python_version: py39 -requirements: [] -resources: - accelerator: H100:8 - cpu: '1' - memory: 10Gi - use_gpu: true -secrets: - hf_access_token: null -system_packages: [] -trt_llm: - build: - base_model: decoder - checkpoint_repository: - repo: meta-llama/Llama-3.1-405B - revision: main - source: HF - max_seq_len: 131072 - plugin_configuration: - use_fp8_context_fmha: true - quantization_type: fp8_kv - tensor_parallel_count: 8 - runtime: - enable_chunked_context: true - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/llama/llama-3-70b-instruct/README.md b/llama/llama-3-70b-instruct/README.md deleted file mode 100644 index 2a81e4680..000000000 --- a/llama/llama-3-70b-instruct/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Llama 3 70B Instruct - -This is a [Truss](https://truss.baseten.co/) for Llama 3 70B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Llama 3 70B. - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp)). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd llama-3-70b-instruct -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `llama-3-70b-instruct` as your working directory, you can deploy the model with: - -```sh -truss push --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Llama 3 70B Instruct API documentation - -This section provides an overview of the Llama 2 7B API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __messages__: The input text that you want the model to generate a response for. -- __max_tokens__ (optional, default=512): The maximum number of tokens to return, counting input tokens. Maximum of 4096. -- __temperature__ (optional, default=0.1): Controls the randomness of the generated text. Higher values produce more diverse results, while lower values produce more deterministic results. -- __top_p__ (optional, default=0.75): The cumulative probability threshold for token sampling. The model will only consider tokens whose cumulative probability is below this threshold. -- __top_k__ (optional, default=40): The number of top tokens to consider when sampling. The model will only consider the top_k highest-probability tokens. -- __num_beams__ (optional, default=4): The number of beams used for beam search. Increasing this value can result in higher-quality output but will increase the computational cost. - -The API also supports passing any parameter supported by HuggingFace's `Transformers.generate`. - -## Example usage - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "messages": [{"role": "user", "content": "What even is AGI?"}], - "max_tokens": 128 - }' -``` diff --git a/llama/llama-3-8b-instruct/README.md b/llama/llama-3-8b-instruct/README.md deleted file mode 100644 index 714a6f689..000000000 --- a/llama/llama-3-8b-instruct/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Llama 3 8B Instruct - -This is a [Truss](https://truss.baseten.co/) for Llama 3 8B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Llama 3 8B. - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp)). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd llama-3-8b-instruct -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `llama-3-8b-instruct` as your working directory, you can deploy the model with: - -```sh -truss push --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Llama 3 8B Instruct API documentation - -This section provides an overview of the Llama 2 7B API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __messages__: The input text that you want the model to generate a response for. -- __max_tokens__ (optional, default=512): The maximum number of tokens to return, counting input tokens. Maximum of 4096. -- __temperature__ (optional, default=0.1): Controls the randomness of the generated text. Higher values produce more diverse results, while lower values produce more deterministic results. -- __top_p__ (optional, default=0.75): The cumulative probability threshold for token sampling. The model will only consider tokens whose cumulative probability is below this threshold. -- __top_k__ (optional, default=40): The number of top tokens to consider when sampling. The model will only consider the top_k highest-probability tokens. -- __num_beams__ (optional, default=4): The number of beams used for beam search. Increasing this value can result in higher-quality output but will increase the computational cost. - -The API also supports passing any parameter supported by HuggingFace's `Transformers.generate`. - -## Example usage - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "messages": [{"role": "user", "content": "What even is AGI?"}], - "max_tokens": 128 - }' -``` diff --git a/llama/llama-3_1-405b-instruct/README.md b/llama/llama-3_1-405b-instruct/README.md deleted file mode 100644 index c4659a033..000000000 --- a/llama/llama-3_1-405b-instruct/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Llama 3.1 405B Instruct - -This is an implementation of Llama 3.1 405B for deployment on Baseten. - -- VLLM for faster inference -- FP8 model weights -- Runs on an 8xH100 instance - -Baseten offers private, secure deployments for LLMs like Llama 3.1 405B, including deployments to your own VPC. -To deploy this model on Baseten, contact us at [support@baseten.co](support@baseten.co). diff --git a/llama/llama-3_2-11b-vision-instruct/README.md b/llama/llama-3_2-11b-vision-instruct/README.md deleted file mode 100644 index 3598138ed..000000000 --- a/llama/llama-3_2-11b-vision-instruct/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# Llama 3.2 11B Vision Instruct vLLM Truss - -This is a [Truss](https://truss.baseten.co/) for Llama 3.2 11B Vision Instruct with vLLM. Llama 3.2 11B Vision Instruct is a multimodal (text + vision) LLM. This README will walk you through how to deploy this Truss on Baseten to get your own instance of it. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd llama/llama-3_2-11b-vision-instruct -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Apply for access to the Llama 3.2 11B Vision Instruct model on hugging face [here](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct). -4. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -5. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_token`. Note that you will *not* be able to successfully deploy this model without doing this. - -With `llama-3_2-11b-vision-instruct` as your working directory, you can deploy the model with: - -```sh -truss push --publish --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Notes - -Limitations from vLLM allow for a maximum of 1 image as input. You will get a memory error otherwise. You can keep track of the issue [here](https://github.com/vllm-project/vllm/issues/8826). - -## Example usage - -```sh -truss predict -d '{model: "llama-3.2-11b-vision-instruct", "messages": [{"role": "user", "content": "Tell me about yourself"}]}' -``` - -Here's another example of invoking your model via a REST API but for image input: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "model: "llama-3.2-11b-vision-instruct", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What type of animal is this? Answer in French only" - }, - { - "type": "image_url", - "image_url": { - "url": "https://vetmed.illinois.edu/wp-content/uploads/2021/04/pc-keller-hedgehog.jpg" - } - } - ] - } - ], - "stream": true, - "max_tokens": 64, - "temperature": 0.2 - }' -``` diff --git a/llava/llava-1.6-sgl/README.md b/llava/llava-1.6-sgl/README.md deleted file mode 100644 index 132775d10..000000000 --- a/llava/llava-1.6-sgl/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# LLaVA v1.6 SGL Truss - -This is a truss to run [Llava 1.6 using SGL](https://github.com/sgl-project/sglang) - -## Deploying LLaVA - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd llava/llava-v1.6-sgl -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `llava/llava-v1.6-sgl` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking LLaVA - -LLaVA takes in the following inputs: - __prompt__(required): Piece of text used as instruction for the LLM - __image__(required): Input image in the form of a base64 string used by the model - __max_new_tokens__(optional): Max number of output tokens generated by the LLM - __temperature__(optional): Configuration for LLM - -LLaVA will respond to the `prompt` conditioned on the `image`. The output is is a stream of tokens containing the model response. - - -```python -from PIL import Image -from io import BytesIO -import base64 -import requests - - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -data = { - "prompt": "What is this a picture of?", - "image": pil_to_b64(Image.open("/path/to/image/mountain.jpeg")), -} - -# Call model endpoint -res = requests.post( - f"https://model-.api.baseten.co/production/predict", - headers=headers, - json=data, - stream=True -) - -# Print the generated tokens as they get streamed -for content in res.iter_content(): - print(content.decode("utf-8"), end="", flush=True) -``` - -Sample Input: -![mountain](https://github.com/basetenlabs/truss-examples/assets/15642666/5eb63370-0296-40ab-9387-428bf5e3cd53) - -Sample output: -``` -This is a picture of Half Dome, a granite dome located in Yosemite National Park in the Sierra Nevada of California. It is one of the most iconic rock formations in the park and a popular destination for hikers and climbers. The image shows the dome with a clear blue sky and some clouds, highlighting the natural beauty of the area. -``` diff --git a/llava/llava-v1.5-7b/README.md b/llava/llava-v1.5-7b/README.md deleted file mode 100644 index 48a685526..000000000 --- a/llava/llava-v1.5-7b/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# LLaVA v1.5 Truss - -This repository packages [LLaVA 1.5](https://github.com/haotian-liu/LLaVA/) as a [Truss](https://truss.baseten.co/). - -LLaVA (Large Language and Vision Assistant) is a highly performant open-source vision language model with capabilities similar to GPT-4V. - -## Deploying LLaVA - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd llava/llava-v1.5-7b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `llava-v1.5-7b` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking LLaVA - -LLaVA takes in two inputs, a `query` and a base64 encoded `image`. LLaVA will respond to the `query` conditioned on the `image`. The output is a JSON blob with a single key, `result`, that answers the `query`. - - -```sh -truss predict -d '{"query": "Describe this picture in detail.", "image": "data:image/png;base64,iVBORw0KGgoA..."}' -``` - -You can also invoke your model via a REST API -``` -curl -X POST https://app.baseten.co/model_versions//predict \ - -H "Content-Type: application/json" \ - -d '{ - "query": "Describe this picture in detail.", - "image": "data:image/png;base64,iVBORw0KGgoA..." - }' -``` diff --git a/llava/llava-v1.6-34b/README.md b/llava/llava-v1.6-34b/README.md deleted file mode 100644 index c8176989d..000000000 --- a/llava/llava-v1.6-34b/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# LLaVA v1.6 Truss - -This repository packages [LLaVA 1.6](https://github.com/haotian-liu/LLaVA/) as a [Truss](https://truss.baseten.co/). - -LLaVA (Large Language and Vision Assistant) is a highly performant open-source vision language model with capabilities similar to GPT-4V. - -## Deploying LLaVA - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd llava/llava-v1.6-34b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `llava-v1.6-34b` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking LLaVA - -LLaVA takes in two inputs, a `query` and a base64 encoded `image`. LLaVA will respond to the `query` conditioned on the `image`. The output is a JSON blob with a single key, `result`, that answers the `query`. - - -```sh -truss predict -f input.json -``` - -You can also invoke your model via a REST API -``` -curl -X POST https://app.baseten.co/model_versions//predict \ - -H "Content-Type: application/json" \ - -d '{ - "query": "Describe this picture in detail.", - "image": "data:image/png;base64,iVBORw0KGgoA..." - }' -``` diff --git a/llava/llava-v1.6-34b/config.yaml b/llava/llava-v1.6-34b/config.yaml deleted file mode 100644 index a5beab7d8..000000000 --- a/llava/llava-v1.6-34b/config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_name: llava-v1.6-34b -python_version: py311 -requirements: -- git+https://github.com/haotian-liu/LLaVA.git -resources: - accelerator: A100 - use_gpu: true -secrets: {} -system_packages: [] diff --git a/llm/README.md b/llm/README.md new file mode 100644 index 000000000..22e6a0e31 --- /dev/null +++ b/llm/README.md @@ -0,0 +1,33 @@ +# Large Language Models + +Production-ready Truss configurations for large language models spanning a wide range of model families, sizes, and serving engines (vLLM, SGLang, TRT-LLM). Many directories contain multiple variants optimized for different hardware or quantization levels. + +| Directory | Models | Description | +|-----------|--------|-------------| +| [llama](llama/) | 14 | Meta Llama 3.x and 4.x models including 8B, 70B, 405B, vision, and TRT-LLM engine builds | +| [qwen](qwen/) | 28 | Alibaba Qwen 2.5 and Qwen 3 models including coder, math, vision, and MoE variants | +| [mistral](mistral/) | 15 | Mistral and Mixtral models with vLLM, TRT-LLM, and Devstral engine builds | +| [deepseek](deepseek/) | 7 | DeepSeek R1 distilled models and vision/OCR variants | +| [nemotron](nemotron/) | 7 | NVIDIA Nemotron models including Nano, Ultra, and vision variants | +| [z-ai](z-ai/) | 5 | Zhipu GLM-4 models in various sizes and quantizations | +| [cogito](cogito/) | 4 | Deep Cogito v2 Preview models on Llama and DeepSeek backbones | +| [gemma](gemma/) | 3 | Google Gemma 2 and 3 models served with vLLM | +| [phi](phi/) | 3 | Microsoft Phi-3 and Phi-3.5 mini instruction-tuned models | +| [llava](llava/) | 3 | LLaVA multimodal vision-language models (v1.5, v1.6) | +| [lora](lora/) | 3 | LoRA adapter serving with vLLM, SGLang, and TRT-LLM engines | +| [falcon](falcon/) | 1 | TII Falcon 3 model with TRT-LLM engine | +| [openai](openai/) | 2 | GPT-OSS 20B and 120B open-source reproductions | +| [minimax](minimax/) | 1 | MiniMax M2-1 model | +| [cogvlm](cogvlm/) | 1 | CogVLM visual question answering model | +| [midnight](midnight/) | 1 | Midnight model for text generation | +| [nsql](nsql/) | 1 | NSQL natural language to SQL model | +| [personaplex-7b-v1](personaplex-7b-v1/) | 1 | PersonaPlex 7B persona-driven chat model | +| [seed](seed/) | 1 | Seed LLM model | + +## Deploying + +Each model can be deployed to Baseten with: + +```bash +truss push +``` diff --git a/llm/cogito/README.md b/llm/cogito/README.md new file mode 100644 index 000000000..968b7fe2c --- /dev/null +++ b/llm/cogito/README.md @@ -0,0 +1,46 @@ +# Cogito v2 Preview Models + +Deploy [Deep Cogito](https://www.deepcogito.com/research/cogito-v2-preview) v2 preview models using vLLM's OpenAI-compatible server. These models feature tool calling and reasoning capabilities built on Llama and DeepSeek architectures. + +| Variant | Size | Architecture | GPU | Path | +|---------|------|--------------|-----|------| +| Cogito v2 Llama 70B | 70B | Dense | 2x H100 | [`cogito-v2-preview-llama-70B-vllm/`](cogito-v2-preview-llama-70B-vllm/) | +| Cogito v2 Llama 109B MoE | 109B | MoE | 4x H100 | [`cogito-v2-preview-llama-109B-MoE-vllm/`](cogito-v2-preview-llama-109B-MoE-vllm/) | +| Cogito v2 Llama 405B | 405B | Dense | 8x B200 | [`cogito-v2-preview-llama-405B-vllm/`](cogito-v2-preview-llama-405B-vllm/) | +| Cogito v2 DeepSeek 671B MoE | 671B | MoE | 8x B200 | [`cogito-v2-preview-deepseek-671B-MoE-vllm/`](cogito-v2-preview-deepseek-671B-MoE-vllm/) | + +## Deploy + +> **Note:** These models require a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push llm/cogito/cogito-v2-preview-llama-70B-vllm +``` + +## Invoke + +All models use the OpenAI ChatCompletion format: + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/environments/production/sync/v1", +) + +response = client.chat.completions.create( + model="deepcogito/cogito-v2-preview-llama-70B", + messages=[{"role": "user", "content": "What is today's temperature in celsius? I'm in Paris."}], + max_tokens=1000, +) + +print(response.choices[0].message.content) +``` + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepcogito/cogito-v2-preview-llama-70B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` diff --git a/llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md b/llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md new file mode 100644 index 000000000..521085c82 --- /dev/null +++ b/llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/README.md @@ -0,0 +1,57 @@ +# Cogito V2 Preview DeepSeek 671B MoE FP8 vLLM + +Deploy [deepcogito/cogito-v2-preview-deepseek-671B-MoE-FP8](https://huggingface.co/deepcogito/cogito-v2-preview-deepseek-671B-MoE-FP8) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepcogito/cogito-v2-preview-deepseek-671B-MoE-FP8](https://huggingface.co/deepcogito/cogito-v2-preview-deepseek-671B-MoE-FP8) | +| Task | Text generation | +| Engine | vLLM | +| GPU | B200:8 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepcogito/cogito-v2-preview-deepseek-671B-MoE-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepcogito/cogito-v2-preview-deepseek-671B-MoE-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.9.2` +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/config.yaml b/llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/config.yaml similarity index 94% rename from cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/config.yaml rename to llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/config.yaml index 03e19c4d2..e84b85284 100644 --- a/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/config.yaml +++ b/llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "deepcogito/cogito-v2-preview-deepseek-671B-MoE-FP8 for text generation" base_image: image: vllm/vllm-openai:v0.9.2 model_metadata: diff --git a/llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md b/llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md new file mode 100644 index 000000000..568ab5357 --- /dev/null +++ b/llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm/README.md @@ -0,0 +1,58 @@ +# Cogito V2 Preview Llama 109B MoE vLLM + +Deploy [deepcogito/cogito-v2-preview-llama-109B-MoE](https://huggingface.co/deepcogito/cogito-v2-preview-llama-109B-MoE) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepcogito/cogito-v2-preview-llama-109B-MoE](https://huggingface.co/deepcogito/cogito-v2-preview-llama-109B-MoE) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:4 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepcogito/cogito-v2-preview-llama-109B-MoE", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepcogito/cogito-v2-preview-llama-109B-MoE", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.10.0` +- Predict concurrency: **32** +- Streaming: **enabled** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/cogito/cogito-v2-preview-llama-109B-MoE-vllm/config.yaml b/llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm/config.yaml similarity index 94% rename from cogito/cogito-v2-preview-llama-109B-MoE-vllm/config.yaml rename to llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm/config.yaml index 83f629b28..ab1430702 100644 --- a/cogito/cogito-v2-preview-llama-109B-MoE-vllm/config.yaml +++ b/llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "deepcogito/cogito-v2-preview-llama-109B-MoE for text generation" base_image: image: vllm/vllm-openai:v0.10.0 model_metadata: diff --git a/llm/cogito/cogito-v2-preview-llama-405B-vllm/README.md b/llm/cogito/cogito-v2-preview-llama-405B-vllm/README.md new file mode 100644 index 000000000..c041e4d3b --- /dev/null +++ b/llm/cogito/cogito-v2-preview-llama-405B-vllm/README.md @@ -0,0 +1,58 @@ +# Cogito V2 Preview Llama 405B vLLM + +Deploy [deepcogito/cogito-v2-preview-llama-405B](https://huggingface.co/deepcogito/cogito-v2-preview-llama-405B) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepcogito/cogito-v2-preview-llama-405B](https://huggingface.co/deepcogito/cogito-v2-preview-llama-405B) | +| Task | Text generation | +| Engine | vLLM | +| GPU | B200:8 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepcogito/cogito-v2-preview-llama-405B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepcogito/cogito-v2-preview-llama-405B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.10.0` +- Predict concurrency: **32** +- Streaming: **enabled** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/cogito/cogito-v2-preview-llama-405B-vllm/config.yaml b/llm/cogito/cogito-v2-preview-llama-405B-vllm/config.yaml similarity index 95% rename from cogito/cogito-v2-preview-llama-405B-vllm/config.yaml rename to llm/cogito/cogito-v2-preview-llama-405B-vllm/config.yaml index 93ea22386..3a39d5d77 100644 --- a/cogito/cogito-v2-preview-llama-405B-vllm/config.yaml +++ b/llm/cogito/cogito-v2-preview-llama-405B-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "deepcogito/cogito-v2-preview-llama-405B for text generation" base_image: image: vllm/vllm-openai:v0.10.0 model_metadata: diff --git a/llm/cogito/cogito-v2-preview-llama-70B-vllm/README.md b/llm/cogito/cogito-v2-preview-llama-70B-vllm/README.md new file mode 100644 index 000000000..54cba8e41 --- /dev/null +++ b/llm/cogito/cogito-v2-preview-llama-70B-vllm/README.md @@ -0,0 +1,58 @@ +# Cogito V2 Preview Llama 70B vLLM + +Deploy [deepcogito/cogito-v2-preview-llama-70B](https://huggingface.co/deepcogito/cogito-v2-preview-llama-70B) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepcogito/cogito-v2-preview-llama-70B](https://huggingface.co/deepcogito/cogito-v2-preview-llama-70B) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepcogito/cogito-v2-preview-llama-70B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepcogito/cogito-v2-preview-llama-70B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.10.0` +- Predict concurrency: **32** +- Streaming: **enabled** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/cogito/cogito-v2-preview-llama-70B-vllm/config.yaml b/llm/cogito/cogito-v2-preview-llama-70B-vllm/config.yaml similarity index 94% rename from cogito/cogito-v2-preview-llama-70B-vllm/config.yaml rename to llm/cogito/cogito-v2-preview-llama-70B-vllm/config.yaml index 9ab76b510..fe75fe12a 100755 --- a/cogito/cogito-v2-preview-llama-70B-vllm/config.yaml +++ b/llm/cogito/cogito-v2-preview-llama-70B-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "deepcogito/cogito-v2-preview-llama-70B for text generation" base_image: image: vllm/vllm-openai:v0.10.0 model_metadata: diff --git a/llm/cogvlm/README.md b/llm/cogvlm/README.md new file mode 100644 index 000000000..4a1708d62 --- /dev/null +++ b/llm/cogvlm/README.md @@ -0,0 +1,29 @@ +# CogVLM + +Deploy CogVLM for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/cogvlm/config.yaml b/llm/cogvlm/config.yaml similarity index 59% rename from cogvlm/config.yaml rename to llm/cogvlm/config.yaml index 83591b309..3e26abc1a 100644 --- a/cogvlm/config.yaml +++ b/llm/cogvlm/config.yaml @@ -1,3 +1,4 @@ +description: "CogVLM for vision-language tasks" environment_variables: {} external_package_dirs: [] model_name: CogVLM @@ -12,6 +13,9 @@ requirements: - Pillow==10.1.0 - xformers==0.0.22 - accelerate==0.25.0 +model_metadata: + repo_id: "THUDM/cogvlm-chat-hf" + example_model_input: {"query": "Describe this image in detail", "image": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"} resources: accelerator: A100 cpu: '3' diff --git a/sana/sana_600M/model/__init__.py b/llm/cogvlm/model/__init__.py similarity index 100% rename from sana/sana_600M/model/__init__.py rename to llm/cogvlm/model/__init__.py diff --git a/cogvlm/model/model.py b/llm/cogvlm/model/model.py similarity index 100% rename from cogvlm/model/model.py rename to llm/cogvlm/model/model.py diff --git a/deepseek-ocr/Bad-Handwriting.png b/llm/deepseek/deepseek-ocr/Bad-Handwriting.png similarity index 100% rename from deepseek-ocr/Bad-Handwriting.png rename to llm/deepseek/deepseek-ocr/Bad-Handwriting.png diff --git a/llm/deepseek/deepseek-ocr/README.md b/llm/deepseek/deepseek-ocr/README.md new file mode 100644 index 000000000..b796c9112 --- /dev/null +++ b/llm/deepseek/deepseek-ocr/README.md @@ -0,0 +1,54 @@ +# deepseek-ocr-latest + +Deploy [deepseek-ai/DeepSeek-OCR](https://huggingface.co/deepseek-ai/DeepSeek-OCR) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-OCR](https://huggingface.co/deepseek-ai/DeepSeek-OCR) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100_40GB | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-OCR", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-OCR", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang@sha256:bb19265cdc61a65a158b84fb69d84f885f4c5f55e12e4515be88223ec067cf50` +- Predict concurrency: **256** diff --git a/deepseek-ocr/config.yaml b/llm/deepseek/deepseek-ocr/config.yaml similarity index 91% rename from deepseek-ocr/config.yaml rename to llm/deepseek/deepseek-ocr/config.yaml index 70f8eddc8..6d629e107 100644 --- a/deepseek-ocr/config.yaml +++ b/llm/deepseek/deepseek-ocr/config.yaml @@ -1,4 +1,6 @@ +description: "DeepSeek OCR for optical character recognition" model_metadata: + repo_id: "deepseek-ai/DeepSeek-OCR" example_model_input: model: "deepseek-ai/DeepSeek-OCR" messages: diff --git a/deepseek-ocr/model/__init__.py b/llm/deepseek/deepseek-ocr/model/__init__.py similarity index 100% rename from deepseek-ocr/model/__init__.py rename to llm/deepseek/deepseek-ocr/model/__init__.py diff --git a/deepseek-ocr/model/model.py b/llm/deepseek/deepseek-ocr/model/model.py similarity index 100% rename from deepseek-ocr/model/model.py rename to llm/deepseek/deepseek-ocr/model/model.py diff --git a/deepseek-ocr/test_document_ocr.py b/llm/deepseek/deepseek-ocr/test_document_ocr.py similarity index 100% rename from deepseek-ocr/test_document_ocr.py rename to llm/deepseek/deepseek-ocr/test_document_ocr.py diff --git a/deepseek-ocr/visualization_1.png b/llm/deepseek/deepseek-ocr/visualization_1.png similarity index 100% rename from deepseek-ocr/visualization_1.png rename to llm/deepseek/deepseek-ocr/visualization_1.png diff --git a/deepseek-ocr/visualization_2.png b/llm/deepseek/deepseek-ocr/visualization_2.png similarity index 100% rename from deepseek-ocr/visualization_2.png rename to llm/deepseek/deepseek-ocr/visualization_2.png diff --git a/deepseek-ocr/visualizer.py b/llm/deepseek/deepseek-ocr/visualizer.py similarity index 100% rename from deepseek-ocr/visualizer.py rename to llm/deepseek/deepseek-ocr/visualizer.py diff --git a/llm/deepseek/deepseek-vl2/README.md b/llm/deepseek/deepseek-vl2/README.md new file mode 100644 index 000000000..19cb0dfda --- /dev/null +++ b/llm/deepseek/deepseek-vl2/README.md @@ -0,0 +1,54 @@ +# deepseek vl2 + +Deploy [deepseek-ai/deepseek-vl2](https://huggingface.co/deepseek-ai/deepseek-vl2) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/deepseek-vl2](https://huggingface.co/deepseek-ai/deepseek-vl2) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/deepseek-vl2", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/deepseek-vl2", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.5.5` +- Predict concurrency: **256** diff --git a/deepseek/deepseek-vl2/config.yaml b/llm/deepseek/deepseek-vl2/config.yaml similarity index 91% rename from deepseek/deepseek-vl2/config.yaml rename to llm/deepseek/deepseek-vl2/config.yaml index 2aee62d71..215398d22 100644 --- a/deepseek/deepseek-vl2/config.yaml +++ b/llm/deepseek/deepseek-vl2/config.yaml @@ -1,4 +1,6 @@ +description: "DeepSeek VL2 for vision-language tasks" model_metadata: + repo_id: "deepseek-ai/deepseek-vl2" example_model_input: model: "deepseek-ai/deepseek-vl2" messages: diff --git a/llm/deepseek/engine-deepseek-r1-distill-llama-70b/README.md b/llm/deepseek/engine-deepseek-r1-distill-llama-70b/README.md new file mode 100644 index 000000000..f80e01a53 --- /dev/null +++ b/llm/deepseek/engine-deepseek-r1-distill-llama-70b/README.md @@ -0,0 +1,65 @@ +# DeepSeek R1 Distill Llama 70B + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **use_fp8_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/deepseek/engine-deepseek-r1-distill-llama-70b/config.yaml b/llm/deepseek/engine-deepseek-r1-distill-llama-70b/config.yaml similarity index 94% rename from deepseek/engine-deepseek-r1-distill-llama-70b/config.yaml rename to llm/deepseek/engine-deepseek-r1-distill-llama-70b/config.yaml index 943f6f9ed..ae853c96d 100644 --- a/deepseek/engine-deepseek-r1-distill-llama-70b/config.yaml +++ b/llm/deepseek/engine-deepseek-r1-distill-llama-70b/config.yaml @@ -1,3 +1,4 @@ +description: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/deepseek/engine-deepseek-r1-distill-llama-8b/README.md b/llm/deepseek/engine-deepseek-r1-distill-llama-8b/README.md new file mode 100644 index 000000000..2804c2ece --- /dev/null +++ b/llm/deepseek/engine-deepseek-r1-distill-llama-8b/README.md @@ -0,0 +1,63 @@ +# DeepSeek R1 Distill Llama 8B + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/deepseek/engine-deepseek-r1-distill-llama-8b/config.yaml b/llm/deepseek/engine-deepseek-r1-distill-llama-8b/config.yaml similarity index 94% rename from deepseek/engine-deepseek-r1-distill-llama-8b/config.yaml rename to llm/deepseek/engine-deepseek-r1-distill-llama-8b/config.yaml index db7448e4c..ccecc83b0 100644 --- a/deepseek/engine-deepseek-r1-distill-llama-8b/config.yaml +++ b/llm/deepseek/engine-deepseek-r1-distill-llama-8b/config.yaml @@ -1,3 +1,4 @@ +description: "deepseek-ai/DeepSeek-R1-Distill-Llama-8B for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/deepseek/engine-deepseek-r1-distill-qwen-14b/README.md b/llm/deepseek/engine-deepseek-r1-distill-qwen-14b/README.md new file mode 100644 index 000000000..c4c52db3c --- /dev/null +++ b/llm/deepseek/engine-deepseek-r1-distill-qwen-14b/README.md @@ -0,0 +1,61 @@ +# DeepSeek R1 Distill Qwen 14B + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Qwen-14B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Qwen-14B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/deepseek/engine-deepseek-r1-distill-qwen-14b/config.yaml b/llm/deepseek/engine-deepseek-r1-distill-qwen-14b/config.yaml similarity index 93% rename from deepseek/engine-deepseek-r1-distill-qwen-14b/config.yaml rename to llm/deepseek/engine-deepseek-r1-distill-qwen-14b/config.yaml index 8a3b06001..c49363fb7 100644 --- a/deepseek/engine-deepseek-r1-distill-qwen-14b/config.yaml +++ b/llm/deepseek/engine-deepseek-r1-distill-qwen-14b/config.yaml @@ -1,3 +1,4 @@ +description: "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/deepseek/engine-deepseek-r1-distill-qwen-32b/README.md b/llm/deepseek/engine-deepseek-r1-distill-qwen-32b/README.md new file mode 100644 index 000000000..beb82dada --- /dev/null +++ b/llm/deepseek/engine-deepseek-r1-distill-qwen-32b/README.md @@ -0,0 +1,61 @@ +# DeepSeek R1 Distill Qwen 32B + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/deepseek/engine-deepseek-r1-distill-qwen-32b/config.yaml b/llm/deepseek/engine-deepseek-r1-distill-qwen-32b/config.yaml similarity index 93% rename from deepseek/engine-deepseek-r1-distill-qwen-32b/config.yaml rename to llm/deepseek/engine-deepseek-r1-distill-qwen-32b/config.yaml index e4eab5457..70260fbfc 100644 --- a/deepseek/engine-deepseek-r1-distill-qwen-32b/config.yaml +++ b/llm/deepseek/engine-deepseek-r1-distill-qwen-32b/config.yaml @@ -1,3 +1,4 @@ +description: "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/deepseek/engine-deepseek-r1-distill-qwen-7b/README.md b/llm/deepseek/engine-deepseek-r1-distill-qwen-7b/README.md new file mode 100644 index 000000000..769e306a3 --- /dev/null +++ b/llm/deepseek/engine-deepseek-r1-distill-qwen-7b/README.md @@ -0,0 +1,61 @@ +# DeepSeek R1 Distill Qwen 7B + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Qwen-7B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Qwen-7B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/deepseek/engine-deepseek-r1-distill-qwen-7b/config.yaml b/llm/deepseek/engine-deepseek-r1-distill-qwen-7b/config.yaml similarity index 93% rename from deepseek/engine-deepseek-r1-distill-qwen-7b/config.yaml rename to llm/deepseek/engine-deepseek-r1-distill-qwen-7b/config.yaml index 5d93c82a1..1bd546125 100644 --- a/deepseek/engine-deepseek-r1-distill-qwen-7b/config.yaml +++ b/llm/deepseek/engine-deepseek-r1-distill-qwen-7b/config.yaml @@ -1,3 +1,4 @@ +description: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/falcon/_archive/falcon3-10B-trt-llm-spec-dec/README.md b/llm/falcon/_archive/falcon3-10B-trt-llm-spec-dec/README.md new file mode 100644 index 000000000..72e9e0944 --- /dev/null +++ b/llm/falcon/_archive/falcon3-10B-trt-llm-spec-dec/README.md @@ -0,0 +1,51 @@ +# Falcon 3 10B Instruct + +Deploy [tiiuae/Falcon3-10B-Instruct](https://huggingface.co/tiiuae/Falcon3-10B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [tiiuae/Falcon3-10B-Instruct](https://huggingface.co/tiiuae/Falcon3-10B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | NO QUANT | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "frequency_penalty": 1, + "max_tokens": 512, + "messages": [ + { + "content": "You are a knowledgable, engaging, biology teacher.", + "role": "system" + }, + { + "content": "What makes falcons effective hunters?", + "role": "user" + } + ], + "stream": true, + "temperature": 0.6 +}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Speculative decoding: **DRAFT_TOKENS_EXTERNAL** +- Max sequence length: **8,192** +- Chunked context: **enabled** +- Plugin: **paged_kv_cache** +- Plugin: **use_paged_context_fmha** +- Streaming: **enabled** diff --git a/falcon/falcon3-10B-trt-llm-spec-dec/config.yaml b/llm/falcon/_archive/falcon3-10B-trt-llm-spec-dec/config.yaml similarity index 100% rename from falcon/falcon3-10B-trt-llm-spec-dec/config.yaml rename to llm/falcon/_archive/falcon3-10B-trt-llm-spec-dec/config.yaml diff --git a/llm/falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md b/llm/falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md new file mode 100644 index 000000000..b50eebeed --- /dev/null +++ b/llm/falcon/falcon3-3B-trt-llm-engine-high-throughput/README.md @@ -0,0 +1,52 @@ +# Falcon 3 3B Instruct + +Deploy [tiiuae/Falcon3-3B-Instruct](https://huggingface.co/tiiuae/Falcon3-3B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [tiiuae/Falcon3-3B-Instruct](https://huggingface.co/tiiuae/Falcon3-3B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | A10G | +| Quantization | NO QUANT | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "frequency_penalty": 1, + "max_tokens": 512, + "messages": [ + { + "content": "You are a knowledgable, engaging, biology teacher.", + "role": "system" + }, + { + "content": "What makes falcons effective hunters?", + "role": "user" + } + ], + "stream": true, + "temperature": 0.6 +}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **8,192** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **paged_kv_cache** +- Plugin: **use_paged_context_fmha** +- Streaming: **enabled** diff --git a/falcon/falcon3-3B-trt-llm-engine-high-throughput/config.yaml b/llm/falcon/falcon3-3B-trt-llm-engine-high-throughput/config.yaml similarity index 91% rename from falcon/falcon3-3B-trt-llm-engine-high-throughput/config.yaml rename to llm/falcon/falcon3-3B-trt-llm-engine-high-throughput/config.yaml index 862407cf1..0d606c4e3 100644 --- a/falcon/falcon3-3B-trt-llm-engine-high-throughput/config.yaml +++ b/llm/falcon/falcon3-3B-trt-llm-engine-high-throughput/config.yaml @@ -1,7 +1,10 @@ +description: "tiiuae/Falcon3-3B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] model_metadata: + tags: + - openai-compatible example_model_input: frequency_penalty: 1 max_tokens: 512 diff --git a/llm/gemma/gemma-2-27b-it-vllm/README.md b/llm/gemma/gemma-2-27b-it-vllm/README.md new file mode 100644 index 000000000..de26a50ee --- /dev/null +++ b/llm/gemma/gemma-2-27b-it-vllm/README.md @@ -0,0 +1,34 @@ +# Gemma 2 27B Instruct VLLM + +Deploy [google/gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [google/gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "what is the meaning of life" +}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/gemma/gemma-2-27b-it-vllm/config.yaml b/llm/gemma/gemma-2-27b-it-vllm/config.yaml similarity index 89% rename from gemma/gemma-2-27b-it-vllm/config.yaml rename to llm/gemma/gemma-2-27b-it-vllm/config.yaml index 26593477f..6efea777c 100644 --- a/gemma/gemma-2-27b-it-vllm/config.yaml +++ b/llm/gemma/gemma-2-27b-it-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "google/gemma-2-27b-it for text generation" model_name: "Gemma 2 27B Instruct VLLM" python_version: py311 model_metadata: diff --git a/sana/sana_600M/packages/Sana/diffusion/model/__init__.py b/llm/gemma/gemma-2-27b-it-vllm/model/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/__init__.py rename to llm/gemma/gemma-2-27b-it-vllm/model/__init__.py diff --git a/gemma/gemma-2-27b-it-vllm/model/model.py b/llm/gemma/gemma-2-27b-it-vllm/model/model.py similarity index 100% rename from gemma/gemma-2-27b-it-vllm/model/model.py rename to llm/gemma/gemma-2-27b-it-vllm/model/model.py diff --git a/llm/gemma/gemma-2-9b-it-vllm/README.md b/llm/gemma/gemma-2-9b-it-vllm/README.md new file mode 100644 index 000000000..761165113 --- /dev/null +++ b/llm/gemma/gemma-2-9b-it-vllm/README.md @@ -0,0 +1,34 @@ +# Gemma 2 9B Instruct VLLM + +Deploy [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "what is the meaning of life" +}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/gemma/gemma-2-9b-it-vllm/config.yaml b/llm/gemma/gemma-2-9b-it-vllm/config.yaml similarity index 89% rename from gemma/gemma-2-9b-it-vllm/config.yaml rename to llm/gemma/gemma-2-9b-it-vllm/config.yaml index d6b1306a5..f0d3f16d5 100644 --- a/gemma/gemma-2-9b-it-vllm/config.yaml +++ b/llm/gemma/gemma-2-9b-it-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "google/gemma-2-9b-it for text generation" model_name: "Gemma 2 9B Instruct VLLM" python_version: py311 model_metadata: diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py b/llm/gemma/gemma-2-9b-it-vllm/model/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/__init__.py rename to llm/gemma/gemma-2-9b-it-vllm/model/__init__.py diff --git a/gemma/gemma-2-9b-it-vllm/model/model.py b/llm/gemma/gemma-2-9b-it-vllm/model/model.py similarity index 100% rename from gemma/gemma-2-9b-it-vllm/model/model.py rename to llm/gemma/gemma-2-9b-it-vllm/model/model.py diff --git a/llm/gemma/gemma-3-27b-it/README.md b/llm/gemma/gemma-3-27b-it/README.md new file mode 100644 index 000000000..c1704c5c4 --- /dev/null +++ b/llm/gemma/gemma-3-27b-it/README.md @@ -0,0 +1,59 @@ +# Gemma 27B Instruct + +Deploy [google/gemma-3-27b-it](https://huggingface.co/google/gemma-3-27b-it) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [google/gemma-3-27b-it](https://huggingface.co/google/gemma-3-27b-it) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="google/gemma-3-27b-it", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "google/gemma-3-27b-it", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:8a4a2efc6fc32cdc30e4e35ba3f8c64dcd0aa1d0` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **8** +- Streaming: **enabled** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/gemma/gemma-3-27b-it/config.yaml b/llm/gemma/gemma-3-27b-it/config.yaml similarity index 93% rename from gemma/gemma-3-27b-it/config.yaml rename to llm/gemma/gemma-3-27b-it/config.yaml index 9c28b9aee..e2b5513f3 100644 --- a/gemma/gemma-3-27b-it/config.yaml +++ b/llm/gemma/gemma-3-27b-it/config.yaml @@ -1,3 +1,4 @@ +description: "google/gemma-3-27b-it for text generation" base_image: image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:8a4a2efc6fc32cdc30e4e35ba3f8c64dcd0aa1d0 build_commands: @@ -43,9 +44,9 @@ model_cache: use_volume: true volume_folder: gemma requirements: -- huggingface_hub -- hf_transfer -- datasets +- huggingface_hub==0.19.4 +- hf_transfer==0.1.4 +- datasets==2.16.1 resources: accelerator: H100 use_gpu: true diff --git a/llama/llama-2-13b-chat/README.md b/llm/llama/_archive/llama-2-13b-chat/README.md similarity index 100% rename from llama/llama-2-13b-chat/README.md rename to llm/llama/_archive/llama-2-13b-chat/README.md diff --git a/llama/llama-2-13b-chat/config.yaml b/llm/llama/_archive/llama-2-13b-chat/config.yaml similarity index 100% rename from llama/llama-2-13b-chat/config.yaml rename to llm/llama/_archive/llama-2-13b-chat/config.yaml diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py b/llm/llama/_archive/llama-2-13b-chat/model/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/apps/__init__.py rename to llm/llama/_archive/llama-2-13b-chat/model/__init__.py diff --git a/llama/llama-2-13b-chat/model/model.py b/llm/llama/_archive/llama-2-13b-chat/model/model.py similarity index 100% rename from llama/llama-2-13b-chat/model/model.py rename to llm/llama/_archive/llama-2-13b-chat/model/model.py diff --git a/llama/llama-2-13b/README.md b/llm/llama/_archive/llama-2-13b/README.md similarity index 100% rename from llama/llama-2-13b/README.md rename to llm/llama/_archive/llama-2-13b/README.md diff --git a/llama/llama-2-13b/config.yaml b/llm/llama/_archive/llama-2-13b/config.yaml similarity index 100% rename from llama/llama-2-13b/config.yaml rename to llm/llama/_archive/llama-2-13b/config.yaml diff --git a/sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py b/llm/llama/_archive/llama-2-13b/model/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/model/dc_ae/efficientvit/models/__init__.py rename to llm/llama/_archive/llama-2-13b/model/__init__.py diff --git a/llama/llama-2-13b/model/model.py b/llm/llama/_archive/llama-2-13b/model/model.py similarity index 100% rename from llama/llama-2-13b/model/model.py rename to llm/llama/_archive/llama-2-13b/model/model.py diff --git a/llama/llama-2-70b-chat/README.md b/llm/llama/_archive/llama-2-70b-chat/README.md similarity index 100% rename from llama/llama-2-70b-chat/README.md rename to llm/llama/_archive/llama-2-70b-chat/README.md diff --git a/llama/llama-2-70b-chat/config.yaml b/llm/llama/_archive/llama-2-70b-chat/config.yaml similarity index 100% rename from llama/llama-2-70b-chat/config.yaml rename to llm/llama/_archive/llama-2-70b-chat/config.yaml diff --git a/sana/sana_600M/packages/Sana/diffusion/utils/__init__.py b/llm/llama/_archive/llama-2-70b-chat/model/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from sana/sana_600M/packages/Sana/diffusion/utils/__init__.py rename to llm/llama/_archive/llama-2-70b-chat/model/__init__.py diff --git a/llama/llama-2-70b-chat/model/model.py b/llm/llama/_archive/llama-2-70b-chat/model/model.py similarity index 100% rename from llama/llama-2-70b-chat/model/model.py rename to llm/llama/_archive/llama-2-70b-chat/model/model.py diff --git a/llama/llama-2-70b/README.md b/llm/llama/_archive/llama-2-70b/README.md similarity index 100% rename from llama/llama-2-70b/README.md rename to llm/llama/_archive/llama-2-70b/README.md diff --git a/llama/llama-2-70b/config.yaml b/llm/llama/_archive/llama-2-70b/config.yaml similarity index 100% rename from llama/llama-2-70b/config.yaml rename to llm/llama/_archive/llama-2-70b/config.yaml diff --git a/sana/sana_600M/packages/Sana/tools/__init__.py b/llm/llama/_archive/llama-2-70b/model/__init__.py similarity index 100% rename from sana/sana_600M/packages/Sana/tools/__init__.py rename to llm/llama/_archive/llama-2-70b/model/__init__.py diff --git a/llama/llama-2-70b/model/model.py b/llm/llama/_archive/llama-2-70b/model/model.py similarity index 100% rename from llama/llama-2-70b/model/model.py rename to llm/llama/_archive/llama-2-70b/model/model.py diff --git a/llama/llama-2-7b-chat/README.md b/llm/llama/_archive/llama-2-7b-chat/README.md similarity index 100% rename from llama/llama-2-7b-chat/README.md rename to llm/llama/_archive/llama-2-7b-chat/README.md diff --git a/llama/llama-2-7b-chat/config.yaml b/llm/llama/_archive/llama-2-7b-chat/config.yaml similarity index 100% rename from llama/llama-2-7b-chat/config.yaml rename to llm/llama/_archive/llama-2-7b-chat/config.yaml diff --git a/segment-anything/model/__init__.py b/llm/llama/_archive/llama-2-7b-chat/model/__init__.py similarity index 100% rename from segment-anything/model/__init__.py rename to llm/llama/_archive/llama-2-7b-chat/model/__init__.py diff --git a/llama/llama-2-7b-chat/model/model.py b/llm/llama/_archive/llama-2-7b-chat/model/model.py similarity index 100% rename from llama/llama-2-7b-chat/model/model.py rename to llm/llama/_archive/llama-2-7b-chat/model/model.py diff --git a/llama/llama-2-7b/README.md b/llm/llama/_archive/llama-2-7b/README.md similarity index 100% rename from llama/llama-2-7b/README.md rename to llm/llama/_archive/llama-2-7b/README.md diff --git a/llama/llama-2-7b/config.yaml b/llm/llama/_archive/llama-2-7b/config.yaml similarity index 100% rename from llama/llama-2-7b/config.yaml rename to llm/llama/_archive/llama-2-7b/config.yaml diff --git a/sesame-csm-1b/model/__init__.py b/llm/llama/_archive/llama-2-7b/model/__init__.py similarity index 100% rename from sesame-csm-1b/model/__init__.py rename to llm/llama/_archive/llama-2-7b/model/__init__.py diff --git a/llama/llama-2-7b/model/model.py b/llm/llama/_archive/llama-2-7b/model/model.py similarity index 100% rename from llama/llama-2-7b/model/model.py rename to llm/llama/_archive/llama-2-7b/model/model.py diff --git a/llama/llama-7b-exllama-streaming/README.md b/llm/llama/_archive/llama-7b-exllama-streaming/README.md similarity index 100% rename from llama/llama-7b-exllama-streaming/README.md rename to llm/llama/_archive/llama-7b-exllama-streaming/README.md diff --git a/llama/llama-7b-exllama-streaming/config.yaml b/llm/llama/_archive/llama-7b-exllama-streaming/config.yaml similarity index 100% rename from llama/llama-7b-exllama-streaming/config.yaml rename to llm/llama/_archive/llama-7b-exllama-streaming/config.yaml diff --git a/stable-diffusion/dreamshaper-lcm/model/__init__.py b/llm/llama/_archive/llama-7b-exllama-streaming/model/__init__.py similarity index 100% rename from stable-diffusion/dreamshaper-lcm/model/__init__.py rename to llm/llama/_archive/llama-7b-exllama-streaming/model/__init__.py diff --git a/llama/llama-7b-exllama-streaming/model/model.py b/llm/llama/_archive/llama-7b-exllama-streaming/model/model.py similarity index 100% rename from llama/llama-7b-exllama-streaming/model/model.py rename to llm/llama/_archive/llama-7b-exllama-streaming/model/model.py diff --git a/llama/llama-7b-exllama/README.md b/llm/llama/_archive/llama-7b-exllama/README.md similarity index 100% rename from llama/llama-7b-exllama/README.md rename to llm/llama/_archive/llama-7b-exllama/README.md diff --git a/llama/llama-7b-exllama/config.yaml b/llm/llama/_archive/llama-7b-exllama/config.yaml similarity index 100% rename from llama/llama-7b-exllama/config.yaml rename to llm/llama/_archive/llama-7b-exllama/config.yaml diff --git a/stable-diffusion/playground-v2-trt/model/__init__.py b/llm/llama/_archive/llama-7b-exllama/model/__init__.py similarity index 100% rename from stable-diffusion/playground-v2-trt/model/__init__.py rename to llm/llama/_archive/llama-7b-exllama/model/__init__.py diff --git a/llama/llama-7b-exllama/model/model.py b/llm/llama/_archive/llama-7b-exllama/model/model.py similarity index 100% rename from llama/llama-7b-exllama/model/model.py rename to llm/llama/_archive/llama-7b-exllama/model/model.py diff --git a/llama/llama-7b-vllm/config.yaml b/llm/llama/_archive/llama-7b-vllm/config.yaml similarity index 100% rename from llama/llama-7b-vllm/config.yaml rename to llm/llama/_archive/llama-7b-vllm/config.yaml diff --git a/stable-diffusion/sd-textual-inversion/model/__init__.py b/llm/llama/_archive/llama-7b-vllm/model/__init__.py similarity index 100% rename from stable-diffusion/sd-textual-inversion/model/__init__.py rename to llm/llama/_archive/llama-7b-vllm/model/__init__.py diff --git a/llama/llama-7b-vllm/model/model.py b/llm/llama/_archive/llama-7b-vllm/model/model.py similarity index 100% rename from llama/llama-7b-vllm/model/model.py rename to llm/llama/_archive/llama-7b-vllm/model/model.py diff --git a/llama/llama-7b/README.md b/llm/llama/_archive/llama-7b/README.md similarity index 100% rename from llama/llama-7b/README.md rename to llm/llama/_archive/llama-7b/README.md diff --git a/llama/llama-7b/config.yaml b/llm/llama/_archive/llama-7b/config.yaml similarity index 100% rename from llama/llama-7b/config.yaml rename to llm/llama/_archive/llama-7b/config.yaml diff --git a/llama/llama-7b/data/config.json b/llm/llama/_archive/llama-7b/data/config.json similarity index 100% rename from llama/llama-7b/data/config.json rename to llm/llama/_archive/llama-7b/data/config.json diff --git a/llama/llama-7b/data/generation_config.json b/llm/llama/_archive/llama-7b/data/generation_config.json similarity index 100% rename from llama/llama-7b/data/generation_config.json rename to llm/llama/_archive/llama-7b/data/generation_config.json diff --git a/llama/llama-7b/data/pytorch_model.bin.index.json b/llm/llama/_archive/llama-7b/data/pytorch_model.bin.index.json similarity index 100% rename from llama/llama-7b/data/pytorch_model.bin.index.json rename to llm/llama/_archive/llama-7b/data/pytorch_model.bin.index.json diff --git a/stable-diffusion/sd-turbo/model/__init__.py b/llm/llama/_archive/llama-7b/model/__init__.py similarity index 100% rename from stable-diffusion/sd-turbo/model/__init__.py rename to llm/llama/_archive/llama-7b/model/__init__.py diff --git a/llama/llama-7b/model/model.py b/llm/llama/_archive/llama-7b/model/model.py similarity index 100% rename from llama/llama-7b/model/model.py rename to llm/llama/_archive/llama-7b/model/model.py diff --git a/llm/llama/engine-llama-3-1-70b-instruct/README.md b/llm/llama/engine-llama-3-1-70b-instruct/README.md new file mode 100644 index 000000000..75cbf93ff --- /dev/null +++ b/llm/llama/engine-llama-3-1-70b-instruct/README.md @@ -0,0 +1,65 @@ +# Llama 3.1 70B Instruct + +Deploy [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.1-70B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-70B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **use_fp8_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/llama/engine-llama-3-1-70b-instruct/config.yaml b/llm/llama/engine-llama-3-1-70b-instruct/config.yaml similarity index 95% rename from llama/engine-llama-3-1-70b-instruct/config.yaml rename to llm/llama/engine-llama-3-1-70b-instruct/config.yaml index 04727e12c..d86a44901 100644 --- a/llama/engine-llama-3-1-70b-instruct/config.yaml +++ b/llm/llama/engine-llama-3-1-70b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-3.1-70B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/llama/engine-llama-3-1-8b-instruct/README.md b/llm/llama/engine-llama-3-1-8b-instruct/README.md new file mode 100644 index 000000000..1c5cadba0 --- /dev/null +++ b/llm/llama/engine-llama-3-1-8b-instruct/README.md @@ -0,0 +1,63 @@ +# Llama 3.1 8B Instruct + +Deploy [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-8B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/llama/engine-llama-3-1-8b-instruct/config.yaml b/llm/llama/engine-llama-3-1-8b-instruct/config.yaml similarity index 95% rename from llama/engine-llama-3-1-8b-instruct/config.yaml rename to llm/llama/engine-llama-3-1-8b-instruct/config.yaml index 3ba5e8020..b9a43b774 100644 --- a/llama/engine-llama-3-1-8b-instruct/config.yaml +++ b/llm/llama/engine-llama-3-1-8b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-3.1-8B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/llama/engine-llama-3-3-70b-instruct/README.md b/llm/llama/engine-llama-3-3-70b-instruct/README.md new file mode 100644 index 000000000..2b9a33ead --- /dev/null +++ b/llm/llama/engine-llama-3-3-70b-instruct/README.md @@ -0,0 +1,65 @@ +# Llama 3.3 70B Instruct + +Deploy [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.3-70B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **use_fp8_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/llama/engine-llama-3-3-70b-instruct/config.yaml b/llm/llama/engine-llama-3-3-70b-instruct/config.yaml similarity index 95% rename from llama/engine-llama-3-3-70b-instruct/config.yaml rename to llm/llama/engine-llama-3-3-70b-instruct/config.yaml index 7492dd6f0..12524bf21 100644 --- a/llama/engine-llama-3-3-70b-instruct/config.yaml +++ b/llm/llama/engine-llama-3-3-70b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-3.3-70B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/llama/engine-llama-3.1-405b-instruct/README.md b/llm/llama/engine-llama-3.1-405b-instruct/README.md new file mode 100644 index 000000000..2a2f8d456 --- /dev/null +++ b/llm/llama/engine-llama-3.1-405b-instruct/README.md @@ -0,0 +1,63 @@ +# Briton-meta-llama-llama-3.1-405b-fp8-truss-example + +Deploy [https://mp-model-weights-public.s3.us-east-2.amazonaws.com/llama-405b-tp8-fp8kv-tllm.tar](https://huggingface.co/https://mp-model-weights-public.s3.us-east-2.amazonaws.com/llama-405b-tp8-fp8kv-tllm.tar) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [https://mp-model-weights-public.s3.us-east-2.amazonaws.com/llama-405b-tp8-fp8kv-tllm.tar](https://huggingface.co/https://mp-model-weights-public.s3.us-east-2.amazonaws.com/llama-405b-tp8-fp8kv-tllm.tar) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:8 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="https://mp-model-weights-public.s3.us-east-2.amazonaws.com/llama-405b-tp8-fp8kv-tllm.tar", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "https://mp-model-weights-public.s3.us-east-2.amazonaws.com/llama-405b-tp8-fp8kv-tllm.tar", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **8** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** +- Environment variables: `ENABLE_EXECUTOR_API` diff --git a/llama/engine-llama-3.1-405b-instruct/config.yaml b/llm/llama/engine-llama-3.1-405b-instruct/config.yaml similarity index 93% rename from llama/engine-llama-3.1-405b-instruct/config.yaml rename to llm/llama/engine-llama-3.1-405b-instruct/config.yaml index f1d063fe9..9eecc559c 100644 --- a/llama/engine-llama-3.1-405b-instruct/config.yaml +++ b/llm/llama/engine-llama-3.1-405b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Briton-meta-llama-llama-3.1-405b-fp8-truss-example for text generation" build_commands: [] environment_variables: ENABLE_EXECUTOR_API: 1 diff --git a/llm/llama/llama-3-1-405b-instruct/README.md b/llm/llama/llama-3-1-405b-instruct/README.md new file mode 100644 index 000000000..0aba694f9 --- /dev/null +++ b/llm/llama/llama-3-1-405b-instruct/README.md @@ -0,0 +1,34 @@ +# Llama 3.1 405B Instruct VLLM + +Deploy [meta-llama/Llama-3.1-405B-Instruct-FP8](https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct-FP8) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-405B-Instruct-FP8](https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct-FP8) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | H100:8 | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "what is the meaning of life" +}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/llama/llama-3_1-405b-instruct/config.yaml b/llm/llama/llama-3-1-405b-instruct/config.yaml similarity index 84% rename from llama/llama-3_1-405b-instruct/config.yaml rename to llm/llama/llama-3-1-405b-instruct/config.yaml index 61d9a3a59..7d7ce18be 100644 --- a/llama/llama-3_1-405b-instruct/config.yaml +++ b/llm/llama/llama-3-1-405b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-3.1-405B-Instruct-FP8 for text generation" model_name: "Llama 3.1 405B Instruct VLLM" python_version: py311 model_metadata: diff --git a/stable-diffusion/sdxl-controlnet-canny/model/__init__.py b/llm/llama/llama-3-1-405b-instruct/model/__init__.py similarity index 100% rename from stable-diffusion/sdxl-controlnet-canny/model/__init__.py rename to llm/llama/llama-3-1-405b-instruct/model/__init__.py diff --git a/llama/llama-3_1-405b-instruct/model/model.py b/llm/llama/llama-3-1-405b-instruct/model/model.py similarity index 100% rename from llama/llama-3_1-405b-instruct/model/model.py rename to llm/llama/llama-3-1-405b-instruct/model/model.py diff --git a/llama/llama-3_1-405b-instruct/model/sighelper.py b/llm/llama/llama-3-1-405b-instruct/model/sighelper.py similarity index 100% rename from llama/llama-3_1-405b-instruct/model/sighelper.py rename to llm/llama/llama-3-1-405b-instruct/model/sighelper.py diff --git a/llm/llama/llama-3-1-70b-instruct/README.md b/llm/llama/llama-3-1-70b-instruct/README.md new file mode 100644 index 000000000..625566964 --- /dev/null +++ b/llm/llama/llama-3-1-70b-instruct/README.md @@ -0,0 +1,32 @@ +# Llama 3.1 70B vLLM + +Deploy Llama 3.1 70B vLLM for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100:4 | +| Python | py310 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Predict concurrency: **128** +- System packages: `python3.10-venv` diff --git a/llama/llama-3_1_70b-instruct/config.yaml b/llm/llama/llama-3-1-70b-instruct/config.yaml similarity index 58% rename from llama/llama-3_1_70b-instruct/config.yaml rename to llm/llama/llama-3-1-70b-instruct/config.yaml index a6776abf7..bc59de7e0 100644 --- a/llama/llama-3_1_70b-instruct/config.yaml +++ b/llm/llama/llama-3-1-70b-instruct/config.yaml @@ -1,11 +1,14 @@ +description: "Llama 3.1 70B vLLM for text generation" environment_variables: {} external_package_dirs: [] -model_metadata: {} +model_metadata: + repo_id: "meta-llama/Llama-3.1-70B-Instruct" + example_model_input: {"prompt": "What is the meaning of life?"} model_name: Llama 3.1 70B vLLM python_version: py310 requirements: - vllm==0.5.3post1 - - accelerate + - accelerate==0.25.0 resources: accelerator: A100:4 use_gpu: true diff --git a/stable-diffusion/sdxl-controlnet-depth/model/__init__.py b/llm/llama/llama-3-1-70b-instruct/model/__init__.py similarity index 100% rename from stable-diffusion/sdxl-controlnet-depth/model/__init__.py rename to llm/llama/llama-3-1-70b-instruct/model/__init__.py diff --git a/llama/llama-3_1_70b-instruct/model/model.py b/llm/llama/llama-3-1-70b-instruct/model/model.py similarity index 100% rename from llama/llama-3_1_70b-instruct/model/model.py rename to llm/llama/llama-3-1-70b-instruct/model/model.py diff --git a/llama/llama-3_1_70b-instruct/model/sighelper.py b/llm/llama/llama-3-1-70b-instruct/model/sighelper.py similarity index 100% rename from llama/llama-3_1_70b-instruct/model/sighelper.py rename to llm/llama/llama-3-1-70b-instruct/model/sighelper.py diff --git a/llm/llama/llama-3-1-8b-instruct-sglang/README.md b/llm/llama/llama-3-1-8b-instruct-sglang/README.md new file mode 100644 index 000000000..2704a90e5 --- /dev/null +++ b/llm/llama/llama-3-1-8b-instruct-sglang/README.md @@ -0,0 +1,34 @@ +# Llama 3.1 8B Instruct SGLang + +Deploy [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100 | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "what is the meaning of life" +}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/llama/llama-3_1-8b-instruct-sglang/config.yaml b/llm/llama/llama-3-1-8b-instruct-sglang/config.yaml similarity index 90% rename from llama/llama-3_1-8b-instruct-sglang/config.yaml rename to llm/llama/llama-3-1-8b-instruct-sglang/config.yaml index dbbd25d85..b55650a94 100644 --- a/llama/llama-3_1-8b-instruct-sglang/config.yaml +++ b/llm/llama/llama-3-1-8b-instruct-sglang/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-3.1-8B-Instruct for text generation" model_name: "Llama 3.1 8B Instruct SGLang" python_version: py311 model_metadata: diff --git a/stable-diffusion/sdxl-controlnet/model/__init__.py b/llm/llama/llama-3-1-8b-instruct-sglang/model/__init__.py similarity index 100% rename from stable-diffusion/sdxl-controlnet/model/__init__.py rename to llm/llama/llama-3-1-8b-instruct-sglang/model/__init__.py diff --git a/llama/llama-3_1-8b-instruct-sglang/model/model.py b/llm/llama/llama-3-1-8b-instruct-sglang/model/model.py similarity index 100% rename from llama/llama-3_1-8b-instruct-sglang/model/model.py rename to llm/llama/llama-3-1-8b-instruct-sglang/model/model.py diff --git a/llm/llama/llama-3-1-8b-instruct/README.md b/llm/llama/llama-3-1-8b-instruct/README.md new file mode 100644 index 000000000..61abf3733 --- /dev/null +++ b/llm/llama/llama-3-1-8b-instruct/README.md @@ -0,0 +1,34 @@ +# Llama 3.1 8B Instruct VLLM + +Deploy [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | H100_40GB | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "what is the meaning of life" +}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/llama/llama-3_1-8b-instruct/config.yaml b/llm/llama/llama-3-1-8b-instruct/config.yaml similarity index 88% rename from llama/llama-3_1-8b-instruct/config.yaml rename to llm/llama/llama-3-1-8b-instruct/config.yaml index 3ab0cadd2..7778ec92b 100644 --- a/llama/llama-3_1-8b-instruct/config.yaml +++ b/llm/llama/llama-3-1-8b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-3.1-8B-Instruct for text generation" model_name: "Llama 3.1 8B Instruct VLLM" python_version: py311 model_metadata: diff --git a/stable-diffusion/sdxl-lightning/model/__init__.py b/llm/llama/llama-3-1-8b-instruct/model/__init__.py similarity index 100% rename from stable-diffusion/sdxl-lightning/model/__init__.py rename to llm/llama/llama-3-1-8b-instruct/model/__init__.py diff --git a/llama/llama-3_1-8b-instruct/model/model.py b/llm/llama/llama-3-1-8b-instruct/model/model.py similarity index 100% rename from llama/llama-3_1-8b-instruct/model/model.py rename to llm/llama/llama-3-1-8b-instruct/model/model.py diff --git a/llm/llama/llama-3-2-11b-vision-instruct/README.md b/llm/llama/llama-3-2-11b-vision-instruct/README.md new file mode 100644 index 000000000..7a6cc6be6 --- /dev/null +++ b/llm/llama/llama-3-2-11b-vision-instruct/README.md @@ -0,0 +1,58 @@ +# Llama 3.2 11B Vision Instruct + +Deploy [meta-llama/Llama-3.2-11B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.2-11B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct) | +| Task | Text generation | +| Engine | vLLM | +| GPU | A100 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.2-11B-Vision-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.2-11B-Vision-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.6.3.post1` +- Predict concurrency: **64** +- Streaming: **enabled** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/llama/llama-3_2-11b-vision-instruct/config.yaml b/llm/llama/llama-3-2-11b-vision-instruct/config.yaml similarity index 95% rename from llama/llama-3_2-11b-vision-instruct/config.yaml rename to llm/llama/llama-3-2-11b-vision-instruct/config.yaml index eaf5a4973..756897ef5 100644 --- a/llama/llama-3_2-11b-vision-instruct/config.yaml +++ b/llm/llama/llama-3-2-11b-vision-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Llama 3.2 11B Vision Instruct for vision-language tasks" base_image: image: vllm/vllm-openai:v0.6.3.post1 model_metadata: diff --git a/llm/llama/llama-3-70b-instruct/README.md b/llm/llama/llama-3-70b-instruct/README.md new file mode 100644 index 000000000..09f81552e --- /dev/null +++ b/llm/llama/llama-3-70b-instruct/README.md @@ -0,0 +1,32 @@ +# Llama 3 70B Instruct + +Deploy [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | H100:2 | +| Python | py310 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/llama/llama-3-70b-instruct/config.yaml b/llm/llama/llama-3-70b-instruct/config.yaml similarity index 65% rename from llama/llama-3-70b-instruct/config.yaml rename to llm/llama/llama-3-70b-instruct/config.yaml index 22f91ab1f..08b49bc23 100644 --- a/llama/llama-3-70b-instruct/config.yaml +++ b/llm/llama/llama-3-70b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Meta-Llama-3-8B-Instruct for text generation" environment_variables: {} external_package_dirs: [] model_metadata: @@ -6,13 +7,14 @@ model_metadata: repo_id: meta-llama/Meta-Llama-3-8B-Instruct tags: - text-generation + example_model_input: {"messages": [{"role": "user", "content": "What is the meaning of life?"}]} model_name: Llama 3 70B Instruct python_version: py310 requirements: - - accelerate - - einops - - transformers - - torch + - accelerate==0.25.0 + - einops==0.7.0 + - transformers==4.36.0 + - torch==2.1.0 resources: accelerator: H100:2 use_gpu: true diff --git a/stable-diffusion/sdxl-lora-swapping/model/__init__.py b/llm/llama/llama-3-70b-instruct/model/__init__.py similarity index 100% rename from stable-diffusion/sdxl-lora-swapping/model/__init__.py rename to llm/llama/llama-3-70b-instruct/model/__init__.py diff --git a/llama/llama-3-70b-instruct/model/model.py b/llm/llama/llama-3-70b-instruct/model/model.py similarity index 100% rename from llama/llama-3-70b-instruct/model/model.py rename to llm/llama/llama-3-70b-instruct/model/model.py diff --git a/llm/llama/llama-3-8b-instruct/README.md b/llm/llama/llama-3-8b-instruct/README.md new file mode 100644 index 000000000..46974edf5 --- /dev/null +++ b/llm/llama/llama-3-8b-instruct/README.md @@ -0,0 +1,32 @@ +# Llama 3 8B Instruct + +Deploy [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py310 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/llama/llama-3-8b-instruct/config.yaml b/llm/llama/llama-3-8b-instruct/config.yaml similarity index 68% rename from llama/llama-3-8b-instruct/config.yaml rename to llm/llama/llama-3-8b-instruct/config.yaml index 938962ff0..3119f054e 100644 --- a/llama/llama-3-8b-instruct/config.yaml +++ b/llm/llama/llama-3-8b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Meta-Llama-3-8B-Instruct for text generation" environment_variables: {} external_package_dirs: [] model_metadata: @@ -6,16 +7,17 @@ model_metadata: repo_id: meta-llama/Meta-Llama-3-8B-Instruct tags: - text-generation + example_model_input: {"messages": [{"role": "user", "content": "What is the meaning of life?"}]} model_name: Llama 3 8B Instruct python_version: py310 model_cache: - repo_id: meta-llama/Meta-Llama-3-8B-Instruct use_volume: false requirements: - - accelerate - - einops - - transformers - - torch + - accelerate==0.25.0 + - einops==0.7.0 + - transformers==4.36.0 + - torch==2.1.0 resources: accelerator: A100 use_gpu: true diff --git a/stable-diffusion/sdxl-lora/model/__init__.py b/llm/llama/llama-3-8b-instruct/model/__init__.py similarity index 100% rename from stable-diffusion/sdxl-lora/model/__init__.py rename to llm/llama/llama-3-8b-instruct/model/__init__.py diff --git a/llama/llama-3-8b-instruct/model/model.py b/llm/llama/llama-3-8b-instruct/model/model.py similarity index 100% rename from llama/llama-3-8b-instruct/model/model.py rename to llm/llama/llama-3-8b-instruct/model/model.py diff --git a/llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/README.md b/llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/README.md new file mode 100644 index 000000000..118bf3e1a --- /dev/null +++ b/llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/README.md @@ -0,0 +1,59 @@ +# Llama 4 Maverick 17B 128E Instruct H100 TP8 + +Deploy [meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:8 | +| Quantization | FP8 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.8.4` +- Predict concurrency: **256** +- Streaming: **enabled** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/config.yaml b/llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/config.yaml similarity index 88% rename from llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/config.yaml rename to llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/config.yaml index f75d0583f..a98aa5e2e 100644 --- a/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/config.yaml +++ b/llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 for text generation" base_image: image: vllm/vllm-openai:v0.8.4 build_commands: @@ -5,7 +6,7 @@ build_commands: model_metadata: repo_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 example_model_input: { - "model": "llama", + "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "messages": [ { "role": "user", diff --git a/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/data/do.sh b/llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/data/do.sh similarity index 100% rename from llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/data/do.sh rename to llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm/data/do.sh diff --git a/llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/README.md b/llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/README.md new file mode 100644 index 000000000..2ba9618d2 --- /dev/null +++ b/llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/README.md @@ -0,0 +1,57 @@ +# Llama 4 Scout 17B 16E Instruct H100 TP 4 + +Deploy [meta-llama/Llama-4-Scout-17B-16E-Instruct](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-4-Scout-17B-16E-Instruct](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:4 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.8.4` +- Predict concurrency: **256** +- Streaming: **enabled** diff --git a/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/config.yaml b/llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/config.yaml similarity index 93% rename from llama/llama-4-scout-17b-16e-instruct-bf16-vllm/config.yaml rename to llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/config.yaml index 96da2d062..c836a3fae 100755 --- a/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/config.yaml +++ b/llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "meta-llama/Llama-4-Scout-17B-16E-Instruct for text generation" base_image: image: vllm/vllm-openai:v0.8.4 build_commands: diff --git a/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/data/do.sh b/llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/data/do.sh similarity index 100% rename from llama/llama-4-scout-17b-16e-instruct-bf16-vllm/data/do.sh rename to llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm/data/do.sh diff --git a/llm/llama/tinyllama-1.1B-chat-v1.0/README.md b/llm/llama/tinyllama-1.1B-chat-v1.0/README.md new file mode 100644 index 000000000..5992e39c9 --- /dev/null +++ b/llm/llama/tinyllama-1.1B-chat-v1.0/README.md @@ -0,0 +1,56 @@ +# tinyllama-trt + +Deploy [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | A10G | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **2,048** diff --git a/llama/tinyllama-1.1B-chat-v1.0/config.yaml b/llm/llama/tinyllama-1.1B-chat-v1.0/config.yaml similarity index 85% rename from llama/tinyllama-1.1B-chat-v1.0/config.yaml rename to llm/llama/tinyllama-1.1B-chat-v1.0/config.yaml index 0d33221e6..24d464ef1 100644 --- a/llama/tinyllama-1.1B-chat-v1.0/config.yaml +++ b/llm/llama/tinyllama-1.1B-chat-v1.0/config.yaml @@ -1,3 +1,4 @@ +description: "TinyLlama/TinyLlama-1.1B-Chat-v1.0 for text generation" model_metadata: tags: - openai-compatible diff --git a/llm/llava/llava-1.6-sgl/README.md b/llm/llava/llava-1.6-sgl/README.md new file mode 100644 index 000000000..320ad500b --- /dev/null +++ b/llm/llava/llava-1.6-sgl/README.md @@ -0,0 +1,29 @@ +# llava 1.6 SGL + +Deploy llava 1.6 SGL for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/llava/llava-1.6-sgl/config.yaml b/llm/llava/llava-1.6-sgl/config.yaml similarity index 54% rename from llava/llava-1.6-sgl/config.yaml rename to llm/llava/llava-1.6-sgl/config.yaml index ae2eb6956..54a0fe5b7 100644 --- a/llava/llava-1.6-sgl/config.yaml +++ b/llm/llava/llava-1.6-sgl/config.yaml @@ -1,9 +1,13 @@ +description: "LLaVA 1.6 for vision-language tasks" environment_variables: {} external_package_dirs: [] model_name: llava 1.6 SGL python_version: py310 requirements: [] requirements_file: ./requirements.txt +model_metadata: + repo_id: "liuhaotian/llava-v1.6-34b" + example_model_input: {"prompt": "Describe this image in detail", "image": "data:image/png;base64,iVBORw0KGgo..."} resources: accelerator: A100 use_gpu: true diff --git a/stable-diffusion/sdxl-turbo/model/__init__.py b/llm/llava/llava-1.6-sgl/model/__init__.py similarity index 100% rename from stable-diffusion/sdxl-turbo/model/__init__.py rename to llm/llava/llava-1.6-sgl/model/__init__.py diff --git a/llava/llava-1.6-sgl/model/model.py b/llm/llava/llava-1.6-sgl/model/model.py similarity index 100% rename from llava/llava-1.6-sgl/model/model.py rename to llm/llava/llava-1.6-sgl/model/model.py diff --git a/llava/llava-1.6-sgl/requirements.txt b/llm/llava/llava-1.6-sgl/requirements.txt similarity index 100% rename from llava/llava-1.6-sgl/requirements.txt rename to llm/llava/llava-1.6-sgl/requirements.txt diff --git a/llm/llava/llava-v1.5-7b/README.md b/llm/llava/llava-v1.5-7b/README.md new file mode 100644 index 000000000..801b5c6c4 --- /dev/null +++ b/llm/llava/llava-v1.5-7b/README.md @@ -0,0 +1,29 @@ +# llava-v1.5-7b + +Deploy llava-v1.5-7b for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/llava/llava-v1.5-7b/config.yaml b/llm/llava/llava-v1.5-7b/config.yaml similarity index 63% rename from llava/llava-v1.5-7b/config.yaml rename to llm/llava/llava-v1.5-7b/config.yaml index 120a72be5..c942f50ed 100644 --- a/llava/llava-v1.5-7b/config.yaml +++ b/llm/llava/llava-v1.5-7b/config.yaml @@ -1,3 +1,4 @@ +description: "LLaVA v1.5 7B for vision-language tasks" environment_variables: {} external_package_dirs: [] model_name: llava-v1.5-7b @@ -16,6 +17,9 @@ requirements: - einops==0.6.1 - einops-exts==0.0.4 - timm==0.6.13 +model_metadata: + repo_id: "liuhaotian/llava-v1.5-7b" + example_model_input: {"query": "Describe this image in detail", "image": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"} resources: accelerator: A10G cpu: '3' diff --git a/stable-diffusion/stable-diffusion-3-medium/model/__init__.py b/llm/llava/llava-v1.5-7b/model/__init__.py similarity index 100% rename from stable-diffusion/stable-diffusion-3-medium/model/__init__.py rename to llm/llava/llava-v1.5-7b/model/__init__.py diff --git a/llava/llava-v1.5-7b/model/model.py b/llm/llava/llava-v1.5-7b/model/model.py similarity index 100% rename from llava/llava-v1.5-7b/model/model.py rename to llm/llava/llava-v1.5-7b/model/model.py diff --git a/llava/llava-v1.5-7b/packages/llava/__init__.py b/llm/llava/llava-v1.5-7b/packages/llava/__init__.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/__init__.py rename to llm/llava/llava-v1.5-7b/packages/llava/__init__.py diff --git a/llava/llava-v1.5-7b/packages/llava/constants.py b/llm/llava/llava-v1.5-7b/packages/llava/constants.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/constants.py rename to llm/llava/llava-v1.5-7b/packages/llava/constants.py diff --git a/llava/llava-v1.5-7b/packages/llava/conversation.py b/llm/llava/llava-v1.5-7b/packages/llava/conversation.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/conversation.py rename to llm/llava/llava-v1.5-7b/packages/llava/conversation.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_bench.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_bench.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_bench.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_bench.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_visual.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_visual.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_visual.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_gpt_review_visual.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_pope.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_pope.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_pope.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_pope.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4_requery.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4_requery.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4_requery.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_science_qa_gpt4_requery.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/eval_textvqa.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/eval_textvqa.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/eval_textvqa.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/eval_textvqa.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/generate_webpage_data_from_table.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/generate_webpage_data_from_table.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/generate_webpage_data_from_table.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/generate_webpage_data_from_table.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/m4c_evaluator.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/m4c_evaluator.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/m4c_evaluator.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/m4c_evaluator.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/model_qa.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/model_qa.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/model_qa.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/model_qa.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/model_vqa.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/model_vqa.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_loader.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_loader.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/model_vqa_loader.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_loader.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_mmbench.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_mmbench.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/model_vqa_mmbench.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_mmbench.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_qbench.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_qbench.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/model_vqa_qbench.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_qbench.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_science.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_science.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/model_vqa_science.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/model_vqa_science.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/qa_baseline_gpt35.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/qa_baseline_gpt35.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/qa_baseline_gpt35.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/qa_baseline_gpt35.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/run_llava.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/run_llava.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/run_llava.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/run_llava.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/summarize_gpt_review.py b/llm/llava/llava-v1.5-7b/packages/llava/eval/summarize_gpt_review.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/summarize_gpt_review.py rename to llm/llava/llava-v1.5-7b/packages/llava/eval/summarize_gpt_review.py diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_alpaca-13b.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_alpaca-13b.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_alpaca-13b.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_alpaca-13b.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_bard.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_bard.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_bard.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_bard.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_gpt35.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_gpt35.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_gpt35.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_gpt35.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_llama-13b.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_llama-13b.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_llama-13b.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_llama-13b.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_vicuna-13b.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_vicuna-13b.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_vicuna-13b.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/answer/answer_vicuna-13b.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/caps_boxes_coco2014_val_80.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/caps_boxes_coco2014_val_80.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/caps_boxes_coco2014_val_80.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/caps_boxes_coco2014_val_80.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/model.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/model.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/model.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/model.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/prompt.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/prompt.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/prompt.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/prompt.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/question.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/question.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/question.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/question.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_13b_v0.json b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_13b_v0.json similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_13b_v0.json rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_13b_v0.json diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_bard_vicuna-13b.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_bard_vicuna-13b.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/review/review_bard_vicuna-13b.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_bard_vicuna-13b.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_gpt35_vicuna-13b.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_gpt35_vicuna-13b.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/review/review_gpt35_vicuna-13b.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_gpt35_vicuna-13b.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/reviewer.jsonl b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/reviewer.jsonl similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/reviewer.jsonl rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/reviewer.jsonl diff --git a/llava/llava-v1.5-7b/packages/llava/eval/table/rule.json b/llm/llava/llava-v1.5-7b/packages/llava/eval/table/rule.json similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/table/rule.json rename to llm/llava/llava-v1.5-7b/packages/llava/eval/table/rule.json diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/alpaca.png b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/alpaca.png similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/alpaca.png rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/alpaca.png diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/bard.jpg b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/bard.jpg similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/bard.jpg rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/bard.jpg diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/chatgpt.svg b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/chatgpt.svg similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/chatgpt.svg rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/chatgpt.svg diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/llama.jpg b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/llama.jpg similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/llama.jpg rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/llama.jpg diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/vicuna.jpeg b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/vicuna.jpeg similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/vicuna.jpeg rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/figures/vicuna.jpeg diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/index.html b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/index.html similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/index.html rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/index.html diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/script.js b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/script.js similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/script.js rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/script.js diff --git a/llava/llava-v1.5-7b/packages/llava/eval/webpage/styles.css b/llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/styles.css similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/eval/webpage/styles.css rename to llm/llava/llava-v1.5-7b/packages/llava/eval/webpage/styles.css diff --git a/llava/llava-v1.5-7b/packages/llava/mm_utils.py b/llm/llava/llava-v1.5-7b/packages/llava/mm_utils.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/mm_utils.py rename to llm/llava/llava-v1.5-7b/packages/llava/mm_utils.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/__init__.py b/llm/llava/llava-v1.5-7b/packages/llava/model/__init__.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/__init__.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/__init__.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/apply_delta.py b/llm/llava/llava-v1.5-7b/packages/llava/model/apply_delta.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/apply_delta.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/apply_delta.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/builder.py b/llm/llava/llava-v1.5-7b/packages/llava/model/builder.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/builder.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/builder.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/consolidate.py b/llm/llava/llava-v1.5-7b/packages/llava/model/consolidate.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/consolidate.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/consolidate.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/llava_llama.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/llava_llama.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/llava_llama.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/llava_llama.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/llava_mpt.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/llava_mpt.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/llava_mpt.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/llava_mpt.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/adapt_tokenizer.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/adapt_tokenizer.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/adapt_tokenizer.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/adapt_tokenizer.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/attention.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/attention.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/attention.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/attention.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/blocks.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/blocks.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/blocks.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/blocks.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/configuration_mpt.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/configuration_mpt.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/configuration_mpt.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/configuration_mpt.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/custom_embedding.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/custom_embedding.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/custom_embedding.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/custom_embedding.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/flash_attn_triton.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/flash_attn_triton.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/flash_attn_triton.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/flash_attn_triton.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/hf_prefixlm_converter.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/hf_prefixlm_converter.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/hf_prefixlm_converter.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/hf_prefixlm_converter.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/meta_init_context.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/meta_init_context.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/meta_init_context.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/meta_init_context.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/modeling_mpt.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/modeling_mpt.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/modeling_mpt.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/modeling_mpt.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/norm.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/norm.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/norm.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/norm.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/param_init_fns.py b/llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/param_init_fns.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/param_init_fns.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/language_model/mpt/param_init_fns.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/llava_arch.py b/llm/llava/llava-v1.5-7b/packages/llava/model/llava_arch.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/llava_arch.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/llava_arch.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/make_delta.py b/llm/llava/llava-v1.5-7b/packages/llava/model/make_delta.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/make_delta.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/make_delta.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/builder.py b/llm/llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/builder.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/builder.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/builder.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/clip_encoder.py b/llm/llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/clip_encoder.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/clip_encoder.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/multimodal_encoder/clip_encoder.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/multimodal_projector/builder.py b/llm/llava/llava-v1.5-7b/packages/llava/model/multimodal_projector/builder.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/multimodal_projector/builder.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/multimodal_projector/builder.py diff --git a/llava/llava-v1.5-7b/packages/llava/model/utils.py b/llm/llava/llava-v1.5-7b/packages/llava/model/utils.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/model/utils.py rename to llm/llava/llava-v1.5-7b/packages/llava/model/utils.py diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/model/__init__.py b/llm/llava/llava-v1.5-7b/packages/llava/serve/__init__.py similarity index 100% rename from stable-diffusion/stable-diffusion-inpainting-trt/model/__init__.py rename to llm/llava/llava-v1.5-7b/packages/llava/serve/__init__.py diff --git a/llava/llava-v1.5-7b/packages/llava/serve/cli.py b/llm/llava/llava-v1.5-7b/packages/llava/serve/cli.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/cli.py rename to llm/llava/llava-v1.5-7b/packages/llava/serve/cli.py diff --git a/llava/llava-v1.5-7b/packages/llava/serve/controller.py b/llm/llava/llava-v1.5-7b/packages/llava/serve/controller.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/controller.py rename to llm/llava/llava-v1.5-7b/packages/llava/serve/controller.py diff --git a/llava/llava-v1.5-7b/packages/llava/serve/examples/extreme_ironing.jpg b/llm/llava/llava-v1.5-7b/packages/llava/serve/examples/extreme_ironing.jpg similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/examples/extreme_ironing.jpg rename to llm/llava/llava-v1.5-7b/packages/llava/serve/examples/extreme_ironing.jpg diff --git a/llava/llava-v1.5-7b/packages/llava/serve/examples/waterview.jpg b/llm/llava/llava-v1.5-7b/packages/llava/serve/examples/waterview.jpg similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/examples/waterview.jpg rename to llm/llava/llava-v1.5-7b/packages/llava/serve/examples/waterview.jpg diff --git a/llava/llava-v1.5-7b/packages/llava/serve/gradio_web_server.py b/llm/llava/llava-v1.5-7b/packages/llava/serve/gradio_web_server.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/gradio_web_server.py rename to llm/llava/llava-v1.5-7b/packages/llava/serve/gradio_web_server.py diff --git a/llava/llava-v1.5-7b/packages/llava/serve/model_worker.py b/llm/llava/llava-v1.5-7b/packages/llava/serve/model_worker.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/model_worker.py rename to llm/llava/llava-v1.5-7b/packages/llava/serve/model_worker.py diff --git a/llava/llava-v1.5-7b/packages/llava/serve/register_worker.py b/llm/llava/llava-v1.5-7b/packages/llava/serve/register_worker.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/register_worker.py rename to llm/llava/llava-v1.5-7b/packages/llava/serve/register_worker.py diff --git a/llava/llava-v1.5-7b/packages/llava/serve/test_message.py b/llm/llava/llava-v1.5-7b/packages/llava/serve/test_message.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/serve/test_message.py rename to llm/llava/llava-v1.5-7b/packages/llava/serve/test_message.py diff --git a/llava/llava-v1.5-7b/packages/llava/train/llama_flash_attn_monkey_patch.py b/llm/llava/llava-v1.5-7b/packages/llava/train/llama_flash_attn_monkey_patch.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/train/llama_flash_attn_monkey_patch.py rename to llm/llava/llava-v1.5-7b/packages/llava/train/llama_flash_attn_monkey_patch.py diff --git a/llava/llava-v1.5-7b/packages/llava/train/llama_xformers_attn_monkey_patch.py b/llm/llava/llava-v1.5-7b/packages/llava/train/llama_xformers_attn_monkey_patch.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/train/llama_xformers_attn_monkey_patch.py rename to llm/llava/llava-v1.5-7b/packages/llava/train/llama_xformers_attn_monkey_patch.py diff --git a/llava/llava-v1.5-7b/packages/llava/train/llava_trainer.py b/llm/llava/llava-v1.5-7b/packages/llava/train/llava_trainer.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/train/llava_trainer.py rename to llm/llava/llava-v1.5-7b/packages/llava/train/llava_trainer.py diff --git a/llava/llava-v1.5-7b/packages/llava/train/train.py b/llm/llava/llava-v1.5-7b/packages/llava/train/train.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/train/train.py rename to llm/llava/llava-v1.5-7b/packages/llava/train/train.py diff --git a/llava/llava-v1.5-7b/packages/llava/train/train_mem.py b/llm/llava/llava-v1.5-7b/packages/llava/train/train_mem.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/train/train_mem.py rename to llm/llava/llava-v1.5-7b/packages/llava/train/train_mem.py diff --git a/llava/llava-v1.5-7b/packages/llava/train/train_xformers.py b/llm/llava/llava-v1.5-7b/packages/llava/train/train_xformers.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/train/train_xformers.py rename to llm/llava/llava-v1.5-7b/packages/llava/train/train_xformers.py diff --git a/llava/llava-v1.5-7b/packages/llava/utils.py b/llm/llava/llava-v1.5-7b/packages/llava/utils.py similarity index 100% rename from llava/llava-v1.5-7b/packages/llava/utils.py rename to llm/llava/llava-v1.5-7b/packages/llava/utils.py diff --git a/llm/llava/llava-v1.6-34b/README.md b/llm/llava/llava-v1.6-34b/README.md new file mode 100644 index 000000000..833f6dccc --- /dev/null +++ b/llm/llava/llava-v1.6-34b/README.md @@ -0,0 +1,29 @@ +# llava-v1.6-34b + +Deploy llava-v1.6-34b for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/llm/llava/llava-v1.6-34b/config.yaml b/llm/llava/llava-v1.6-34b/config.yaml new file mode 100644 index 000000000..705068009 --- /dev/null +++ b/llm/llava/llava-v1.6-34b/config.yaml @@ -0,0 +1,15 @@ +description: "LLaVA v1.6 34B for vision-language tasks" +environment_variables: {} +external_package_dirs: [] +model_name: llava-v1.6-34b +python_version: py311 +requirements: +- git+https://github.com/haotian-liu/LLaVA.git +model_metadata: + repo_id: "liuhaotian/llava-v1.6-34b" + example_model_input: {"query": "Describe this image in detail", "image": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"} +resources: + accelerator: A100 + use_gpu: true +secrets: {} +system_packages: [] diff --git a/llava/llava-v1.6-34b/input.json b/llm/llava/llava-v1.6-34b/input.json similarity index 100% rename from llava/llava-v1.6-34b/input.json rename to llm/llava/llava-v1.6-34b/input.json diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/__init__.py b/llm/llava/llava-v1.6-34b/model/__init__.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt-h100/model/__init__.py rename to llm/llava/llava-v1.6-34b/model/__init__.py diff --git a/llava/llava-v1.6-34b/model/model.py b/llm/llava/llava-v1.6-34b/model/model.py similarity index 100% rename from llava/llava-v1.6-34b/model/model.py rename to llm/llava/llava-v1.6-34b/model/model.py diff --git a/lora/README.md b/llm/lora/README.md similarity index 100% rename from lora/README.md rename to llm/lora/README.md diff --git a/llm/lora/engine-lora/README.md b/llm/lora/engine-lora/README.md new file mode 100644 index 000000000..47d51e546 --- /dev/null +++ b/llm/lora/engine-lora/README.md @@ -0,0 +1,63 @@ +# Mistral 7B Instruct Engine Lora + +Deploy [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mistral-7B-Instruct-v0.3", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mistral-7B-Instruct-v0.3", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/lora/engine-lora/config.yaml b/llm/lora/engine-lora/config.yaml similarity index 95% rename from lora/engine-lora/config.yaml rename to llm/lora/engine-lora/config.yaml index 55dd7908d..82f10314d 100644 --- a/lora/engine-lora/config.yaml +++ b/llm/lora/engine-lora/config.yaml @@ -1,3 +1,4 @@ +description: "Mistral 7B with LoRA adapters via TRT-LLM engine" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/lora/sglang-lora/README.md b/llm/lora/sglang-lora/README.md new file mode 100644 index 000000000..b6297dfed --- /dev/null +++ b/llm/lora/sglang-lora/README.md @@ -0,0 +1,45 @@ +# Mistral-7B-Instruct SGLang Lora + +Deploy [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100_40GB | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/generate \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "text": [ + "What would you choose in 2008?", + "What would you choose in 2008?" + ], + "sampling_params": { + "max_new_tokens": 1000, + "temperature": 1.0 + }, + "lora_path": [ + "legal", + "finance" + ] +}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.9.post6-cu126` +- Predict concurrency: **32** diff --git a/lora/sglang-lora/config.yaml b/llm/lora/sglang-lora/config.yaml similarity index 93% rename from lora/sglang-lora/config.yaml rename to llm/lora/sglang-lora/config.yaml index 9d64c5fd9..ffedd5ce1 100644 --- a/lora/sglang-lora/config.yaml +++ b/llm/lora/sglang-lora/config.yaml @@ -1,3 +1,4 @@ +description: "Mistral 7B with LoRA adapters via SGLang" base_image: image: lmsysorg/sglang:v0.4.9.post6-cu126 model_metadata: @@ -27,4 +28,4 @@ model_name: Mistral-7B-Instruct SGLang Lora environment_variables: hf_access_token: null requirements: - - protobuf + - protobuf==4.25.1 diff --git a/llm/lora/vllm-lora/README.md b/llm/lora/vllm-lora/README.md new file mode 100644 index 000000000..326166da2 --- /dev/null +++ b/llm/lora/vllm-lora/README.md @@ -0,0 +1,44 @@ +# Mistral-7B-Instruct VLLM Lora + +Deploy [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100_40GB | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "finance", + "messages": [ + { + "role": "user", + "content": "How would you choose back in 2008?" + } + ], + "stream": true, + "max_tokens": 512, + "temperature": 0.9 +}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.9.2` +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/lora/vllm-lora/config.yaml b/llm/lora/vllm-lora/config.yaml similarity index 95% rename from lora/vllm-lora/config.yaml rename to llm/lora/vllm-lora/config.yaml index a6e9f84d6..be1b786db 100644 --- a/lora/vllm-lora/config.yaml +++ b/llm/lora/vllm-lora/config.yaml @@ -1,3 +1,4 @@ +description: "Mistral 7B with LoRA adapters via vLLM" base_image: image: vllm/vllm-openai:v0.9.2 model_metadata: diff --git a/llm/midnight/README.md b/llm/midnight/README.md new file mode 100644 index 000000000..1444a676a --- /dev/null +++ b/llm/midnight/README.md @@ -0,0 +1,34 @@ +# Kaiko Midnight + +Pathology foundation model for medical image analysis and classification + +| Property | Value | +|----------|-------| +| Task | Embeddings | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "image_url": "https://upload.wikimedia.org/wikipedia/commons/8/80/Breast_DCIS_histopathology_%281%29.jpg", + "task": "classification", + "batch_size": 1 +}' +``` + +## Configuration highlights + +- Base image: `nvcr.io/nvidia/pytorch:25.06-py3` +- Predict concurrency: **32** diff --git a/midnight/config.yaml b/llm/midnight/config.yaml similarity index 95% rename from midnight/config.yaml rename to llm/midnight/config.yaml index fdfbdaf0b..f4ae65273 100644 --- a/midnight/config.yaml +++ b/llm/midnight/config.yaml @@ -12,6 +12,7 @@ resources: runtime: predict_concurrency: 32 model_metadata: + repo_id: "kaiko-ai/midnight" example_model_input: image_url: "https://upload.wikimedia.org/wikipedia/commons/8/80/Breast_DCIS_histopathology_%281%29.jpg" task: "classification" # or "segmentation" diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt/model/__init__.py b/llm/midnight/model/__init__.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0-trt/model/__init__.py rename to llm/midnight/model/__init__.py diff --git a/midnight/model/model.py b/llm/midnight/model/model.py similarity index 100% rename from midnight/model/model.py rename to llm/midnight/model/model.py diff --git a/midnight/requirements.txt b/llm/midnight/requirements.txt similarity index 100% rename from midnight/requirements.txt rename to llm/midnight/requirements.txt diff --git a/llm/minimax/minimax-m2-1/README.md b/llm/minimax/minimax-m2-1/README.md new file mode 100644 index 000000000..cbe54b415 --- /dev/null +++ b/llm/minimax/minimax-m2-1/README.md @@ -0,0 +1,56 @@ +# minimax + +Deploy [MiniMaxAI/MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [MiniMaxAI/MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:8 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="MiniMaxAI/MiniMax-M2.1", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "MiniMaxAI/MiniMax-M2.1", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:nightly-dev-20260126-48f4340b` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/minimax/minimax_m2_1/config.yaml b/llm/minimax/minimax-m2-1/config.yaml similarity index 96% rename from minimax/minimax_m2_1/config.yaml rename to llm/minimax/minimax-m2-1/config.yaml index b241509ba..cd4719ee9 100644 --- a/minimax/minimax_m2_1/config.yaml +++ b/llm/minimax/minimax-m2-1/config.yaml @@ -1,3 +1,4 @@ +description: "MiniMaxAI/MiniMax-M2.1 for text generation" base_image: image: lmsysorg/sglang:nightly-dev-20260126-48f4340b docker_server: diff --git a/mistral/engine-mistral-7b-instruct/README.md b/llm/mistral/_archive/engine-mistral-7b-instruct/README.md similarity index 100% rename from mistral/engine-mistral-7b-instruct/README.md rename to llm/mistral/_archive/engine-mistral-7b-instruct/README.md diff --git a/mistral/engine-mistral-7b-instruct/config.yaml b/llm/mistral/_archive/engine-mistral-7b-instruct/config.yaml similarity index 100% rename from mistral/engine-mistral-7b-instruct/config.yaml rename to llm/mistral/_archive/engine-mistral-7b-instruct/config.yaml diff --git a/mistral/mistral-7b-chat/README.md b/llm/mistral/_archive/mistral-7b-chat/README.md similarity index 100% rename from mistral/mistral-7b-chat/README.md rename to llm/mistral/_archive/mistral-7b-chat/README.md diff --git a/mistral/mistral-7b-chat/config.yaml b/llm/mistral/_archive/mistral-7b-chat/config.yaml similarity index 100% rename from mistral/mistral-7b-chat/config.yaml rename to llm/mistral/_archive/mistral-7b-chat/config.yaml diff --git a/stable-diffusion/stable-diffusion-xl-1.0/model/__init__.py b/llm/mistral/_archive/mistral-7b-chat/model/__init__.py similarity index 100% rename from stable-diffusion/stable-diffusion-xl-1.0/model/__init__.py rename to llm/mistral/_archive/mistral-7b-chat/model/__init__.py diff --git a/mistral/mistral-7b-chat/model/model.py b/llm/mistral/_archive/mistral-7b-chat/model/model.py similarity index 100% rename from mistral/mistral-7b-chat/model/model.py rename to llm/mistral/_archive/mistral-7b-chat/model/model.py diff --git a/mistral/mistral-7b-instruct-vllm/README.md b/llm/mistral/_archive/mistral-7b-instruct-vllm/README.md similarity index 100% rename from mistral/mistral-7b-instruct-vllm/README.md rename to llm/mistral/_archive/mistral-7b-instruct-vllm/README.md diff --git a/mistral/mistral-7b-instruct-vllm/config.yaml b/llm/mistral/_archive/mistral-7b-instruct-vllm/config.yaml similarity index 100% rename from mistral/mistral-7b-instruct-vllm/config.yaml rename to llm/mistral/_archive/mistral-7b-instruct-vllm/config.yaml diff --git a/stable-diffusion/stable-diffusion/model/__init__.py b/llm/mistral/_archive/mistral-7b-instruct-vllm/model/__init__.py similarity index 100% rename from stable-diffusion/stable-diffusion/model/__init__.py rename to llm/mistral/_archive/mistral-7b-instruct-vllm/model/__init__.py diff --git a/mistral/mistral-7b-instruct-vllm/model/model.py b/llm/mistral/_archive/mistral-7b-instruct-vllm/model/model.py similarity index 100% rename from mistral/mistral-7b-instruct-vllm/model/model.py rename to llm/mistral/_archive/mistral-7b-instruct-vllm/model/model.py diff --git a/mistral/mistral-7b-instruct/README.md b/llm/mistral/_archive/mistral-7b-instruct/README.md similarity index 100% rename from mistral/mistral-7b-instruct/README.md rename to llm/mistral/_archive/mistral-7b-instruct/README.md diff --git a/mistral/mistral-7b-instruct/config.yaml b/llm/mistral/_archive/mistral-7b-instruct/config.yaml similarity index 100% rename from mistral/mistral-7b-instruct/config.yaml rename to llm/mistral/_archive/mistral-7b-instruct/config.yaml diff --git a/stable-diffusion/stable-video-diffusion/model/__init__.py b/llm/mistral/_archive/mistral-7b-instruct/model/__init__.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/__init__.py rename to llm/mistral/_archive/mistral-7b-instruct/model/__init__.py diff --git a/mistral/mistral-7b-instruct/model/model.py b/llm/mistral/_archive/mistral-7b-instruct/model/model.py similarity index 100% rename from mistral/mistral-7b-instruct/model/model.py rename to llm/mistral/_archive/mistral-7b-instruct/model/model.py diff --git a/llm/mistral/engine-devstral/README.md b/llm/mistral/engine-devstral/README.md new file mode 100644 index 000000000..9b9a4dfe1 --- /dev/null +++ b/llm/mistral/engine-devstral/README.md @@ -0,0 +1,60 @@ +# Devstral Small 2505 + +Deploy [mistralai/Devstral-Small-2505](https://huggingface.co/mistralai/Devstral-Small-2505) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Devstral-Small-2505](https://huggingface.co/mistralai/Devstral-Small-2505) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Devstral-Small-2505", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Devstral-Small-2505", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/mistral/engine-devstral/config.yaml b/llm/mistral/engine-devstral/config.yaml similarity index 95% rename from mistral/engine-devstral/config.yaml rename to llm/mistral/engine-devstral/config.yaml index 76a59c3d4..e8596e3e0 100644 --- a/mistral/engine-devstral/config.yaml +++ b/llm/mistral/engine-devstral/config.yaml @@ -1,3 +1,4 @@ +description: "Devstral Small for code generation" model_metadata: example_model_input: { messages: [ diff --git a/llm/mistral/engine-mistral-small-3/README.md b/llm/mistral/engine-mistral-small-3/README.md new file mode 100644 index 000000000..a11c49f08 --- /dev/null +++ b/llm/mistral/engine-mistral-small-3/README.md @@ -0,0 +1,64 @@ +# Mistral Small 3 Instruct FP8 + +Deploy [mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mistral-Small-24B-Instruct-2501", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mistral-Small-24B-Instruct-2501", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **use_fp8_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/mistral/engine-mistral-small-3/config.yaml b/llm/mistral/engine-mistral-small-3/config.yaml similarity index 94% rename from mistral/engine-mistral-small-3/config.yaml rename to llm/mistral/engine-mistral-small-3/config.yaml index 0032b2815..1ee570079 100644 --- a/mistral/engine-mistral-small-3/config.yaml +++ b/llm/mistral/engine-mistral-small-3/config.yaml @@ -1,3 +1,4 @@ +description: "mistralai/Mistral-Small-24B-Instruct-2501 for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/mistral/engine-mixtral-8x22b-instruct/README.md b/llm/mistral/engine-mixtral-8x22b-instruct/README.md new file mode 100644 index 000000000..e7dc3f019 --- /dev/null +++ b/llm/mistral/engine-mixtral-8x22b-instruct/README.md @@ -0,0 +1,60 @@ +# Mistral 8x22B Instruct + +Deploy [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mixtral-8x22B-Instruct-v0.1", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mixtral-8x22B-Instruct-v0.1", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **8,192** +- Streaming: **enabled** diff --git a/mistral/engine-mixtral-8x22b-instruct/config.yaml b/llm/mistral/engine-mixtral-8x22b-instruct/config.yaml similarity index 93% rename from mistral/engine-mixtral-8x22b-instruct/config.yaml rename to llm/mistral/engine-mixtral-8x22b-instruct/config.yaml index f5574031b..c568025cc 100644 --- a/mistral/engine-mixtral-8x22b-instruct/config.yaml +++ b/llm/mistral/engine-mixtral-8x22b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "mistralai/Mixtral-8x22B-Instruct-v0.1 for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/mistral/engine-mixtral-8x7b-instruct/README.md b/llm/mistral/engine-mixtral-8x7b-instruct/README.md new file mode 100644 index 000000000..a71ce490e --- /dev/null +++ b/llm/mistral/engine-mixtral-8x7b-instruct/README.md @@ -0,0 +1,64 @@ +# Mistral 8x7B Instruct + +Deploy [mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mixtral-8x7B-Instruct-v0.1", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mixtral-8x7B-Instruct-v0.1", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **use_fp8_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/mistral/engine-mixtral-8x7b-instruct/config.yaml b/llm/mistral/engine-mixtral-8x7b-instruct/config.yaml similarity index 94% rename from mistral/engine-mixtral-8x7b-instruct/config.yaml rename to llm/mistral/engine-mixtral-8x7b-instruct/config.yaml index c1f992e5c..5e8892002 100644 --- a/mistral/engine-mixtral-8x7b-instruct/config.yaml +++ b/llm/mistral/engine-mixtral-8x7b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "mistralai/Mixtral-8x7B-Instruct-v0.1 for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/mistral/mistral-7b/README.md b/llm/mistral/mistral-7b/README.md new file mode 100644 index 000000000..aa0ad9156 --- /dev/null +++ b/llm/mistral/mistral-7b/README.md @@ -0,0 +1,33 @@ +# mistral-7b + +Deploy mistral-7b for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "What is the Mistral wind?" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/mistral/mistral-7b/config.yaml b/llm/mistral/mistral-7b/config.yaml similarity index 82% rename from mistral/mistral-7b/config.yaml rename to llm/mistral/mistral-7b/config.yaml index ce64c041b..c4291badf 100644 --- a/mistral/mistral-7b/config.yaml +++ b/llm/mistral/mistral-7b/config.yaml @@ -1,6 +1,8 @@ +description: "mistral-7b for text generation" environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "mistralai/Mistral-7B-v0.1" avatar_url: https://cdn.baseten.co/production/static/explore/mistral_logo.png cover_image_url: https://cdn.baseten.co/production/static/explore/mistral.png example_model_input: @@ -12,8 +14,8 @@ model_name: mistral-7b python_version: py311 requirements: - transformers==4.42.3 -- sentencepiece -- accelerate +- sentencepiece==0.1.99 +- accelerate==0.25.0 - torch==2.0.1 - numpy==1.26.4 resources: diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/__init__.py b/llm/mistral/mistral-7b/model/__init__.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/__init__.py rename to llm/mistral/mistral-7b/model/__init__.py diff --git a/mistral/mistral-7b/model/model.py b/llm/mistral/mistral-7b/model/model.py similarity index 100% rename from mistral/mistral-7b/model/model.py rename to llm/mistral/mistral-7b/model/model.py diff --git a/llm/mistral/mistral-small-3.1/README.md b/llm/mistral/mistral-small-3.1/README.md new file mode 100644 index 000000000..ff9cd21f3 --- /dev/null +++ b/llm/mistral/mistral-small-3.1/README.md @@ -0,0 +1,58 @@ +# Mistral Small 3.1 + +Deploy [mistralai/Mistral-Small-3.1-24B-Instruct-2503](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mistral-Small-3.1-24B-Instruct-2503](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:1 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mistral-Small-3.1-24B-Instruct-2503", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mistral-Small-3.1-24B-Instruct-2503", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.10.1.1` +- Predict concurrency: **8** +- Streaming: **enabled** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/mistral/mistral-small-3.1/config.yaml b/llm/mistral/mistral-small-3.1/config.yaml similarity index 91% rename from mistral/mistral-small-3.1/config.yaml rename to llm/mistral/mistral-small-3.1/config.yaml index 159319966..d3dc77068 100644 --- a/mistral/mistral-small-3.1/config.yaml +++ b/llm/mistral/mistral-small-3.1/config.yaml @@ -1,4 +1,5 @@ #vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice --limit_mm_per_prompt 'image=10' --tensor-parallel-size 2 +description: "mistralai/Mistral-Small-3.1-24B-Instruct-2503 for text generation" base_image: image: vllm/vllm-openai:v0.10.1.1 model_metadata: @@ -38,10 +39,10 @@ environment_variables: VLLM_LOGGING_LEVEL: INFO hf_access_token: null requirements: -- huggingface_hub -- hf_transfer -- datasets -- b10-transfer +- huggingface_hub==0.19.4 +- hf_transfer==0.1.4 +- datasets==2.16.1 +- b10-transfer==0.0.5 resources: accelerator: H100:1 use_gpu: true diff --git a/llm/mistral/mixtral-8x22b-trt-int8-weights-only/README.md b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/README.md new file mode 100644 index 000000000..73d3ad4cd --- /dev/null +++ b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/README.md @@ -0,0 +1,57 @@ +# Mixtral 8x22B Instruct TRT-LLM Weights Only Quantized + +Mixtral 8x22B Instruct, with INT8 weights only quantization, optimized with TRT-LLM! + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100:4 | +| Quantization | INT8 | +| OpenAI compatible | Yes | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="model", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "model", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `docker.io/baseten/triton_trt_llm:4062d46_20240401` +- Predict concurrency: **256** diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/TRT-LLM-README.md b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/TRT-LLM-README.md similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/TRT-LLM-README.md rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/TRT-LLM-README.md diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/config.yaml b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/config.yaml similarity index 94% rename from mistral/mixtral-8x22b-trt-int8-weights-only/config.yaml rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/config.yaml index c6b5e6a8b..55e61dc74 100644 --- a/mistral/mixtral-8x22b-trt-int8-weights-only/config.yaml +++ b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/config.yaml @@ -6,6 +6,7 @@ description: Mixtral 8x22B Instruct, with INT8 weights only quantization, optimi environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "mistralai/Mixtral-8x22B-Instruct-v0.1" avatar_url: https://cdn.baseten.co/production/static/explore/mistral_logo.png cover_image_url: https://cdn.baseten.co/production/static/explore/mistral.png engine_repository: baseten/mixtral-8x22B_i60000_o4000_bs2_tp4_int8_weights_only_A100-tllm_0.9.0.dev2024022000 @@ -28,7 +29,7 @@ model_metadata: model_name: Mixtral 8x22B Instruct TRT-LLM Weights Only Quantized python_version: py311 requirements: -- tritonclient[all] +- tritonclient[all]==2.42.0 - transformers==4.42.3 resources: accelerator: A100:4 diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/data/.gitattributes b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/data/.gitattributes similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/data/.gitattributes rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/data/.gitattributes diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/util/__init__.py b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/model/__init__.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/util/__init__.py rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/model/__init__.py diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/model/model.py b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/model/model.py similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/model/model.py rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/model/model.py diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/client.py b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/client.py similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/client.py rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/client.py diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/ensemble/config.pbtxt b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/ensemble/config.pbtxt similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/ensemble/config.pbtxt rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/ensemble/config.pbtxt diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/1/model.py b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/1/model.py rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/1/model.py diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/config.pbtxt b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/config.pbtxt rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/postprocessing/config.pbtxt diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/1/model.py b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/1/model.py rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/1/model.py diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/config.pbtxt b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/config.pbtxt rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/preprocessing/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/utils.py b/llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/utils.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/utils.py rename to llm/mistral/mixtral-8x22b-trt-int8-weights-only/packages/utils.py diff --git a/llm/mistral/mixtral-8x22b/README.md b/llm/mistral/mixtral-8x22b/README.md new file mode 100644 index 000000000..0595821b0 --- /dev/null +++ b/llm/mistral/mixtral-8x22b/README.md @@ -0,0 +1,34 @@ +# Mixtral 8x22 + +Deploy [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100:4 | +| Python | py310 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "What is the Mistral wind?" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/mistral/mixtral-8x22b/config.yaml b/llm/mistral/mixtral-8x22b/config.yaml similarity index 86% rename from mistral/mixtral-8x22b/config.yaml rename to llm/mistral/mixtral-8x22b/config.yaml index d469b4b35..15307ea68 100644 --- a/mistral/mixtral-8x22b/config.yaml +++ b/llm/mistral/mixtral-8x22b/config.yaml @@ -1,3 +1,4 @@ +description: "mistralai/Mixtral-8x22B-Instruct-v0.1 for text generation" environment_variables: {} external_package_dirs: [] model_metadata: @@ -12,7 +13,7 @@ model_metadata: model_name: Mixtral 8x22 python_version: py310 requirements: - - accelerate + - accelerate==0.25.0 - transformers==4.42.3 - torch==2.2.0 resources: diff --git a/stable-diffusion/stable-video-diffusion/model/scripts/util/detection/__init__.py b/llm/mistral/mixtral-8x22b/model/__init__.py similarity index 100% rename from stable-diffusion/stable-video-diffusion/model/scripts/util/detection/__init__.py rename to llm/mistral/mixtral-8x22b/model/__init__.py diff --git a/mistral/mixtral-8x22b/model/model.py b/llm/mistral/mixtral-8x22b/model/model.py similarity index 100% rename from mistral/mixtral-8x22b/model/model.py rename to llm/mistral/mixtral-8x22b/model/model.py diff --git a/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md new file mode 100644 index 000000000..fd159da11 --- /dev/null +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md @@ -0,0 +1,55 @@ +# Mixtral 8x7B Instruct TRT-LLM for H100 + +Mixtral 8x7B Instruct optimized with TRT-LLM! Compatible with OpenAI Client + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | H100:2 | +| OpenAI compatible | Yes | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="model", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "model", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `docker.io/baseten/trtllm-server:r23.12_baseten_v0.7.1` +- Predict concurrency: **256** +- Environment variables: `HF_HUB_ENABLE_HF_TRANSFER` diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/TRT-LLM-README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/TRT-LLM-README.md similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/TRT-LLM-README.md rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/TRT-LLM-README.md diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/config.yaml b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/config.yaml similarity index 94% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/config.yaml rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/config.yaml index 8889d1a52..838027a75 100644 --- a/mistral/mixtral-8x7b-instruct-trt-llm-h100/config.yaml +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/config.yaml @@ -7,6 +7,7 @@ environment_variables: HF_HUB_ENABLE_HF_TRANSFER: 1 external_package_dirs: [] model_metadata: + repo_id: "mistralai/Mixtral-8x7B-Instruct-v0.1" avatar_url: https://cdn.baseten.co/production/static/explore/mistral_logo.png cover_image_url: https://cdn.baseten.co/production/static/explore/mistral.png engine_repository: baseten/mixtral-h100-0.7.1 @@ -29,7 +30,7 @@ model_metadata: model_name: Mixtral 8x7B Instruct TRT-LLM for H100 python_version: py311 requirements: -- tritonclient[all] +- tritonclient[all]==2.42.0 - transformers==4.42.3 - jinja2==3.1.3 - hf_transfer==0.1.5 diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/data/.gitattributes b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/data/.gitattributes similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/data/.gitattributes rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/data/.gitattributes diff --git a/templates/faster-whisper-truss/model/__init__.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/model/__init__.py similarity index 100% rename from templates/faster-whisper-truss/model/__init__.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/model/__init__.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/model/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/model/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/client.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/client.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/client.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/client.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/utils.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/utils.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/utils.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-h100/packages/utils.py diff --git a/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md new file mode 100644 index 000000000..2507441d0 --- /dev/null +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md @@ -0,0 +1,55 @@ +# Mixtral 8x7B Instruct TRT-LLM Weights Only Quantized for H100 + +Mixtral 8x7B Instruct, with INT8 weights only quantization, optimized with TRT-LLM! Compatible with OpenAI Client + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | H100 | +| OpenAI compatible | Yes | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="model", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "model", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `docker.io/baseten/trtllm-server:r23.12_baseten_v0.7.1` +- Predict concurrency: **256** +- Environment variables: `HF_HUB_ENABLE_HF_TRANSFER` diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/TRT-LLM-README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/TRT-LLM-README.md similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/TRT-LLM-README.md rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/TRT-LLM-README.md diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/config.yaml b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/config.yaml similarity index 94% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/config.yaml rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/config.yaml index 68a244d8e..2868bb999 100644 --- a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/config.yaml +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/config.yaml @@ -7,6 +7,7 @@ environment_variables: HF_HUB_ENABLE_HF_TRANSFER: 1 external_package_dirs: [] model_metadata: + repo_id: "mistralai/Mixtral-8x7B-Instruct-v0.1" avatar_url: https://cdn.baseten.co/production/static/explore/mistral_logo.png cover_image_url: https://cdn.baseten.co/production/static/explore/mistral.png engine_repository: baseten/mixtral-weights-only-quantized-h100-0.7.1 @@ -29,7 +30,7 @@ model_metadata: model_name: Mixtral 8x7B Instruct TRT-LLM Weights Only Quantized for H100 python_version: py311 requirements: -- tritonclient[all] +- tritonclient[all]==2.42.0 - transformers==4.42.3 - jinja2==3.1.3 - hf_transfer==0.1.5 diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/data/.gitattributes b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/data/.gitattributes similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/data/.gitattributes rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/data/.gitattributes diff --git a/templates/transformers-openai-compatible/model/__init__.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/__init__.py similarity index 100% rename from templates/transformers-openai-compatible/model/__init__.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/__init__.py diff --git a/templates/trt-llm/model/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/model.py similarity index 100% rename from templates/trt-llm/model/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/model/model.py diff --git a/templates/trt-llm/packages/client.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/client.py similarity index 100% rename from templates/trt-llm/packages/client.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/client.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/ensemble/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/postprocessing/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/preprocessing/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/utils.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/utils.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/utils.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/packages/utils.py diff --git a/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md new file mode 100644 index 000000000..b44d0b754 --- /dev/null +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md @@ -0,0 +1,57 @@ +# Mixtral 8x7B Instruct TRT-LLM Weights Only Quantized + +Mixtral 8x7B Instruct, with INT8 weights only quantization, optimized with TRT-LLM! Compatible with OpenAI Client + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mixtral-8x7B-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| OpenAI compatible | Yes | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mixtral-8x7B-v0.1", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mixtral-8x7B-v0.1", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `docker.io/baseten/triton_trt_llm:main-20231215` +- Predict concurrency: **256** diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/TRT-LLM-README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/TRT-LLM-README.md similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/TRT-LLM-README.md rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/TRT-LLM-README.md diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml similarity index 98% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml index cfafdd705..4fd886745 100644 --- a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/config.yaml @@ -29,7 +29,7 @@ model_metadata: model_name: Mixtral 8x7B Instruct TRT-LLM Weights Only Quantized python_version: py311 requirements: -- tritonclient[all] +- tritonclient[all]==2.42.0 - transformers==4.42.3 resources: accelerator: A100 diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/data/.gitattributes b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/data/.gitattributes similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/data/.gitattributes rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/data/.gitattributes diff --git a/templates/trt-llm/model/__init__.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/__init__.py similarity index 100% rename from templates/trt-llm/model/__init__.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/__init__.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/model/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/client.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/client.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/client.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/client.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/ensemble/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/ensemble/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/ensemble/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/ensemble/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/postprocessing/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/1/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/preprocessing/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/utils.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/utils.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/utils.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/packages/utils.py diff --git a/llm/mistral/mixtral-8x7b-instruct-trt-llm/README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm/README.md new file mode 100644 index 000000000..9b191755b --- /dev/null +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm/README.md @@ -0,0 +1,54 @@ +# Mixtral 8x7B Instruct TRT-LLM + +Mixtral 8x7B Instruct optimized with TRT-LLM! Compatible with OpenAI Client + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100:2 | +| OpenAI compatible | Yes | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="model", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "model", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `docker.io/baseten/triton_trt_llm:main-20231215` +- Predict concurrency: **256** diff --git a/templates/trt-llm/TRT-LLM-README.md b/llm/mistral/mixtral-8x7b-instruct-trt-llm/TRT-LLM-README.md similarity index 100% rename from templates/trt-llm/TRT-LLM-README.md rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/TRT-LLM-README.md diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/config.yaml b/llm/mistral/mixtral-8x7b-instruct-trt-llm/config.yaml similarity index 94% rename from mistral/mixtral-8x7b-instruct-trt-llm/config.yaml rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/config.yaml index 15bff9bab..39a343bf7 100644 --- a/mistral/mixtral-8x7b-instruct-trt-llm/config.yaml +++ b/llm/mistral/mixtral-8x7b-instruct-trt-llm/config.yaml @@ -6,6 +6,7 @@ description: Mixtral 8x7B Instruct optimized with TRT-LLM! Compatible with OpenA environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "mistralai/Mixtral-8x7B-Instruct-v0.1" avatar_url: https://cdn.baseten.co/production/static/explore/mistral_logo.png cover_image_url: https://cdn.baseten.co/production/static/explore/mistral.png engine_repository: baseten/mixtral @@ -28,7 +29,7 @@ model_metadata: model_name: Mixtral 8x7B Instruct TRT-LLM python_version: py311 requirements: -- tritonclient[all] +- tritonclient[all]==2.42.0 - transformers==4.42.3 resources: accelerator: A100:2 diff --git a/templates/trt-llm/data/.gitattributes b/llm/mistral/mixtral-8x7b-instruct-trt-llm/data/.gitattributes similarity index 100% rename from templates/trt-llm/data/.gitattributes rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/data/.gitattributes diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/model/__init__.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm/model/__init__.py similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/high_throughput/model/__init__.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/model/__init__.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/model/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm/model/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/model/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/model/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/client.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/client.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/client.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/client.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/ensemble/config.pbtxt diff --git a/templates/trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py similarity index 100% rename from templates/trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/postprocessing/config.pbtxt diff --git a/templates/trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py similarity index 100% rename from templates/trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/1/model.py diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt similarity index 100% rename from mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/preprocessing/config.pbtxt diff --git a/templates/trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt similarity index 100% rename from templates/trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/inflight_batcher_llm/tensorrt_llm/config.pbtxt diff --git a/templates/trt-llm/packages/utils.py b/llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/utils.py similarity index 100% rename from templates/trt-llm/packages/utils.py rename to llm/mistral/mixtral-8x7b-instruct-trt-llm/packages/utils.py diff --git a/llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md b/llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md new file mode 100644 index 000000000..92667b52c --- /dev/null +++ b/llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md @@ -0,0 +1,29 @@ +# Mixtral 8x7B β€” VLLM TP2 β€” A100:2 + +Deploy Mixtral 8x7B β€” VLLM TP2 β€” A100:2 for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100:2 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/config.yaml b/llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/config.yaml similarity index 53% rename from mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/config.yaml rename to llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/config.yaml index ccc476bea..70305d99a 100644 --- a/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/config.yaml +++ b/llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/config.yaml @@ -1,9 +1,13 @@ +description: "Mixtral 8x7B β€” VLLM TP2 β€” A100:2 for text generation" environment_variables: {} external_package_dirs: [] model_name: Mixtral 8x7B β€” VLLM TP2 β€” A100:2 python_version: py310 requirements: -- vllm +- vllm==0.5.3.post1 +model_metadata: + repo_id: "mistralai/Mixtral-8x7B-Instruct-v0.1" + example_model_input: {"prompt": "What is machine learning?"} resources: accelerator: A100:2 use_gpu: true diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/model/__init__.py b/llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/__init__.py similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/large_context/model/__init__.py rename to llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/__init__.py diff --git a/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/model.py b/llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/model.py rename to llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/model/model.py diff --git a/llm/mistral/mixtral-8x7b-instruct-vllm/README.md b/llm/mistral/mixtral-8x7b-instruct-vllm/README.md new file mode 100644 index 000000000..054520085 --- /dev/null +++ b/llm/mistral/mixtral-8x7b-instruct-vllm/README.md @@ -0,0 +1,29 @@ +# Mixtral 8x7B + +Deploy Mixtral 8x7B for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100:2 | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/mistral/mixtral-8x7b-instruct-vllm/config.yaml b/llm/mistral/mixtral-8x7b-instruct-vllm/config.yaml similarity index 57% rename from mistral/mixtral-8x7b-instruct-vllm/config.yaml rename to llm/mistral/mixtral-8x7b-instruct-vllm/config.yaml index f219fc452..1078b91b4 100644 --- a/mistral/mixtral-8x7b-instruct-vllm/config.yaml +++ b/llm/mistral/mixtral-8x7b-instruct-vllm/config.yaml @@ -1,9 +1,13 @@ +description: "Mixtral 8x7B for text generation" environment_variables: {} external_package_dirs: [] model_name: Mixtral 8x7B python_version: py310 requirements: - vllm==0.2.5 +model_metadata: + repo_id: "mistralai/Mixtral-8x7B-Instruct-v0.1" + example_model_input: {"prompt": "What is machine learning?"} resources: accelerator: A100:2 use_gpu: true diff --git a/trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/model/__init__.py b/llm/mistral/mixtral-8x7b-instruct-vllm/model/__init__.py similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-70b-instruct/low_ttft/model/__init__.py rename to llm/mistral/mixtral-8x7b-instruct-vllm/model/__init__.py diff --git a/mistral/mixtral-8x7b-instruct-vllm/model/model.py b/llm/mistral/mixtral-8x7b-instruct-vllm/model/model.py similarity index 100% rename from mistral/mixtral-8x7b-instruct-vllm/model/model.py rename to llm/mistral/mixtral-8x7b-instruct-vllm/model/model.py diff --git a/llm/mistral/pixtral-12b/README.md b/llm/mistral/pixtral-12b/README.md new file mode 100644 index 000000000..2e6afc2df --- /dev/null +++ b/llm/mistral/pixtral-12b/README.md @@ -0,0 +1,53 @@ +# Pixtral 12B + +Deploy [mistral-community/pixtral-12b-240910](https://huggingface.co/mistral-community/pixtral-12b-240910) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistral-community/pixtral-12b-240910](https://huggingface.co/mistral-community/pixtral-12b-240910) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A100 | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe this image in one sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://picsum.photos/id/237/200/300" + } + } + ] + } + ], + "stream": false, + "max_tokens": 512, + "temperature": 0.5 +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/mistral/pixtral-12b/config.yaml b/llm/mistral/pixtral-12b/config.yaml similarity index 94% rename from mistral/pixtral-12b/config.yaml rename to llm/mistral/pixtral-12b/config.yaml index 5e1d0dba8..cc1ea8a0d 100644 --- a/mistral/pixtral-12b/config.yaml +++ b/llm/mistral/pixtral-12b/config.yaml @@ -1,3 +1,4 @@ +description: "Pixtral 12B for vision-language tasks" model_metadata: repo_id: mistral-community/pixtral-12b-240910 avatar_url: https://cdn.baseten.co/production/static/explore/mistral_logo.png diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/model/__init__.py b/llm/mistral/pixtral-12b/model/__init__.py similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/high_throughput/model/__init__.py rename to llm/mistral/pixtral-12b/model/__init__.py diff --git a/mistral/pixtral-12b/model/model.py b/llm/mistral/pixtral-12b/model/model.py similarity index 100% rename from mistral/pixtral-12b/model/model.py rename to llm/mistral/pixtral-12b/model/model.py diff --git a/llm/nemotron/llama-3-1-nemotron-70b-instruct/README.md b/llm/nemotron/llama-3-1-nemotron-70b-instruct/README.md new file mode 100644 index 000000000..0aa316d21 --- /dev/null +++ b/llm/nemotron/llama-3-1-nemotron-70b-instruct/README.md @@ -0,0 +1,49 @@ +# Llama-3.1-Nemotron-70B-Instruct + +Deploy [nvidia/Llama-3.1-Nemotron-70B-Instruct-HF](https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/Llama-3.1-Nemotron-70B-Instruct-HF](https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 KV | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "user", + "content": "How many r in strawberry?" + } + ], + "stream": true, + "max_tokens": 512, + "temperature": 0.6 +}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **use_fp8_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/nemotron/llama-3-1-nemotron-70b-instruct/config.yaml b/llm/nemotron/llama-3-1-nemotron-70b-instruct/config.yaml similarity index 90% rename from nemotron/llama-3-1-nemotron-70b-instruct/config.yaml rename to llm/nemotron/llama-3-1-nemotron-70b-instruct/config.yaml index a56d044fa..90953b26d 100644 --- a/nemotron/llama-3-1-nemotron-70b-instruct/config.yaml +++ b/llm/nemotron/llama-3-1-nemotron-70b-instruct/config.yaml @@ -1,7 +1,10 @@ +description: "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] model_metadata: + tags: + - openai-compatible example_model_input: { messages: [{ role: "user", content: "How many r in strawberry?" }], diff --git a/llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1/README.md b/llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1/README.md new file mode 100644 index 000000000..150a57989 --- /dev/null +++ b/llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1/README.md @@ -0,0 +1,57 @@ +# Llama 3.1 Nemotron Nano VL 8B V1 + +Deploy [nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1](https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1](https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:1 | +| OpenAI compatible | Yes | +| Python | py312 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.11.0` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **16** +- Streaming: **enabled** diff --git a/nemotron/Llama-3-1-Nemotron-Nano-VL-8B-V1/config.yaml b/llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1/config.yaml similarity index 88% rename from nemotron/Llama-3-1-Nemotron-Nano-VL-8B-V1/config.yaml rename to llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1/config.yaml index f4d752aa4..4ad38520a 100644 --- a/nemotron/Llama-3-1-Nemotron-Nano-VL-8B-V1/config.yaml +++ b/llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1/config.yaml @@ -1,3 +1,4 @@ +description: "Llama 3.1 Nemotron Nano VL 8B for vision-language tasks" base_image: image: vllm/vllm-openai:v0.11.0 model_metadata: @@ -20,11 +21,11 @@ model_metadata: model_name: Llama 3.1 Nemotron Nano VL 8B V1 requirements: - transformers>=4.55.0 - - accelerate - - timm - - einops - - open-clip-torch - - pillow + - accelerate==1.2.1 + - timm==1.0.12 + - einops==0.8.0 + - open-clip-torch==2.29.0 + - pillow==10.4.0 python_version: py312 model_cache: - repo_id: nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1 diff --git a/llm/nemotron/llama-nemoretriever-colembed-3b-v1/README.md b/llm/nemotron/llama-nemoretriever-colembed-3b-v1/README.md new file mode 100644 index 000000000..e19ffc5b7 --- /dev/null +++ b/llm/nemotron/llama-nemoretriever-colembed-3b-v1/README.md @@ -0,0 +1,61 @@ +# Llama NemoRetriever ColEmbed 3B V1 + +NVIDIA's ColEmbed cross-modal embedding model for text and image retrieval + +| Property | Value | +|----------|-------| +| Model | [nvidia/llama-nemoretriever-colembed-3b-v1](https://huggingface.co/nvidia/llama-nemoretriever-colembed-3b-v1) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | L4 | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "queries": [ + "What is machine learning?", + "What is the capital of France?" + ], + "passages": [ + "Machine learning is a subset of artificial intelligence.", + "The capital of France is Paris.", + { + "type": "image", + "content": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxMTEhUTEhMVFRUXFxUXFxgYGBgYFhgdFxgWGRYZFxcaHyggGh0lHhYXITEhJSkrLi4uFx8zODMsNygtLysBCgoKDg0OGBAQGy0dHSUtLS0rLS0tKy0tLS0tLS0rLS0tLS0tKy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tKy0tLf/AABEIALEBHAMBIgACEQEDEQH/xAAbAAACAwEBAQAAAAAAAAAAAAAEBQIDBgEAB//EAEgQAAIBAwMCBAIGBgcGBAcAAAECEQADIQQSMQVBEyJRYQZxFDJCgZGhFSMzscHRUlNicrPC8DRDc4Ky4RaSovEHJDVEY4OT/8QAGAEAAwEBAAAAAAAAAAAAAAAAAAECAwT/xAAjEQACAgIDAQACAwEAAAAAAAAAAQIREiEDMVFBE2EicYEy/9oADAMBAAIRAxEAPwD6DrmJJYeme340oXcxgfdXL+pk5/8Aeu6Fv1in0rsSpHM3bNNY0yWrbZJJj5z91eS4oxyfWqLuu2iIz3oD6eOIj3rNRbNW0g/Vaog1Zo7qt9YSKUX9Wp49e9DjUQcGKvC0Rns0+tccjFKbWv2mRQTdQJ5M1UdYD86a46JlyDm1qt5G7iimue+O1Z9dZV30otzxQ4bBcg9sopmfau2tAAxJ79qU3tfttXDtJhD7fnTO4lxiYU8ms2qNFTGVqyqiIHrQdxVjyc+lBXVuj2++gpuehpxj9sTl8od2tTAg81Yt7cQBQWlJOCD6UZZ0Z7EVLopNjCzaC0FrzmIx60Yz1RrcDMRWcey30KXq5c9oqDQea9YPIitmZILswP59wKNt3MUCxWMVBtURUNWaXRZr9SKC8X1/OuXroJoS43pmrjEhyCNqHtQWr0wnymp+G/YTU00rfa8vzquie/gtKOpmonVXBOD+FNXcL3of6QJk5p2TX7Fha56NUbGmdjgfjTc60ThefeqzqfaJ5p2/BUvSixYKGZFM9Pd7mlbagTxNRfWntScWylJIb60bs7u2BSu9dAMmDHrVP0t27GKqZCeQaSjQOVkbXUNrE9iDXf0s3rVR0s9iK6dAPWqpCTYcOk7uGg+8VfoOmqD52/lRli2AJkk8g/8AajbelXb5myeI7Vk5s0UEL7+lH2Z/P91BPpj3FNrlsAwZHv61y2EGSZpqQnFMT29NuIULJJgAUd+gSAS4j5GaYprgkAQP9Zr13qAjIJ/d91POQlCP0Gt9DsRkt+NUfoq0D9oiiU1IP1MY+4ffVVx27mhOXoNR8JnTWguFE/vqjUEAgDiouxNVMpq0iWxR/wDELWMNIotsVVn23IIyNshSCJOc/dTz4c6i7aWy11ibhRSxJBJ9yR6jtWc+Mgfo4HEuP+k/zpl0YE6eyc/s07ewpYoFJo0hubsxIqhV5waDTePssPuNF2N/IBn8KmqKTstt4z5oo63f4oW1ddvsn8KtKmM4+dQ9lrQd4sCh9VqFIIOaoIHrQ5RTNJRG5FN+8O1UfSmHarTZjjFC3FPpNbKjFsI+mMewrg1frFCS3YVUyNTxROTDzqVnMfxrja4DAAil3htXfo7/ACp4oM2G/pEjiqX1ZNUjSN7UVY0sc0UkCcmVDNEJaBxFXLbHpU2s45iobLUSP0e2BJ5oNlHaimsk4Jqv6N70kNgLaf3rtu2Bzmr7liPWKoYCrJObqibleMVzcPSigsrdjVW40SWFQ3UAMRcEYNXWtS3pPsKoSysc15EXuYqKRdtFmq1ZIiINAknsDRm1PWfvqxtvYimnRL2Brp3ImPzonS6YEebHvNdCwe5++K5duL6R99N2xUkMjqVCQIoK+R60EXFdDD0oUKBzstKnkV5agrek1Lwz8qokR/GjjwreftjsTk4HBHrTn4Xb/wCWteyAfhSL40VgibSsyMsu4eaQBHucffTX4YaNOgbkCDAgcA4H31LLRpFWYMVagFBBgBz+deOrrPFmloY7f7UUHdt5+tNU+PUvFoUWhOSZMWfevG2BXPHFUXb08RT2JtEjznihrkGuNJrq2WiYqyGwh9OoURz60JcT3qbBwKqa2x9aaQmzyIZ5FTkVAaZj/wC9eOnI70aErLBUgaHz61zdRQ7DVep75oAXKt+kiMRUuJSkEVBnihm1FUtep4icwm480Oy1A3a4bw9adCyR1rVVOgHeuPfqprtOmJtEqjvFVtd96pa5TxJyC0LDuasDGrUq2aVjopDVYGq9R8q7FFjor3n0rviGphasC+wpBTB91SRoq6B6V2iwosTUCAP3CprcjnM1RvNRZqmi8jP/ABg4MD5Sds9j/P8AOi/hBp04wMR2jtH+WknxXrE3Mu5d4IgEt6EAmB6rxNMPhC+NhClWG0GVJbIgkZA7XPzFOgbNMbf9kVzw/wCwK6jD5V3fSFo4LQ/o14oK8WroNGwOFK5935VKa5NMCO8+n5Vw3T6VOa4DQKzioTUvoh9astvU9wqW2WkgXwiOCakLBPJNWE+tRZadhRVcsgVSEBq4oa5tpiKGtelQNuiK41OyWgN0zioMtGFagy07JxAGqsmjmtg9qrNgVWROLAWqsmjX04qlrFVaJxYK1QNFGyK9sosKY0VasVKxo6vqRnxmxGCqk+4Mr/qPaujq+rj9sf6U/q+I7jbj76jFl5I2oSpBKxyda1cL+tBxP1EJI94GfnVn6Z1n9JfT9mmfy57RRix5I2ASphaxtv4g1U82/vRP4H5/hVi9f1fPkj/hrH86WDDNGuIrkVlT1/UiCQn/APP/AL1IfEt/+jb/APKw7fOjBhmjTlahcgAkzABOOcDt70hX4jvjm0hByI3DGR6nuOfavanr5ZCGslc5O/8Ao5M4+X40YtApJsxGv6de87O6sBAZnw26N0cROePei/guzct6i3u8ZVOCAPIQwIG7HEx7mPaiLmqD6UeRg1x3uEzP1iNoz6AR91WjVIEsPDibZE5Am2xGPkp/H7qFfQ3Rv1X2qwClGm+IbLKpk7mjEfa4IEnOaJPXLAwXP4flIxNKmGSGG2u7KXDrljHmOSe3pPofap/pvT/1n/pNKmPKIcErwSg16xYP+9UfMMP4VYvUrP8AWp+J/lSpjtBO0VEpVR19rvcUfj/KufpC1/Wr+f8AKimFotg10Gh/p9r+tX8/5V36ba/rF/GnTFaL2rhJqn6Za/rU/Gu/Srf9Yn4x+ZpUO0TrlR+kW/6xP/Ov86811By6f+Zf50BaONXCaru620s7r1oRz51kTngGeKHt9X07YF+3niTt9vtAU6C0E1BqG1PV7CCTdU+gSWJ7wIxx70MnxBpj/vCv95G/KJp0JtBxFRpPqPii2B5Lbt8yFH4iZoS78UPB22lB9SxMfdj99OibQ/YVBlrJXfiHUx9ZRk8LbKxAAHeMgnJ71S3xFqQMkHvPhpn8BFVTJtGuZKgVrKt8TaggAKhMjIWSfYrMfh6VAfE+oP2LfplSPy3U6YWi3SBDcRNmCjMLqnJ2bi20cQFHtXGYW3Cpd5B+yR6BiQCTMGc0DoL1ryhgSxZp9SAw9Ss4nEAmZmi9RbTcEsuFO0bhdCypJEnyyTgFe0TNZZG+KCtLeXeiKWbeYuSIJWfMCy8QwMR7UVYVGnaXXaGY+eSQCcKIwYj5UCga2zFWtCCSGk7TMmCNwJ/AUHrerhYJa2BCiURzMySMHntn5UW30xUl2htYt7vMLj4wQWgndIUY5yO9Wr06wXYHxTCllAZYxEkkiRE8QaX2uootu3dCq3jPsBQAb2BLAHz+UASJj7UxirdXrrasNzgFjJIYO47hIQBSvHBnOaiU2vpShGjvULIs7Z77olvNieccAyJ7xRWqtW7dpHh4cI3PYg9oz+NLdV1pFuSqNdLMDuJTBkkNgTIkgcjiRzXtNqfEDXEuNJBBDoAZkbly2RmZHahTv6Jxj4Ok0bAlN0MrthnAA4I4njmDxPvSrqPS77FpvAgySu6FgnHDwZxijdFpXJZFvo91myIwQIEgASeSeZgVG4nhOj3EueJ54uOriym0+Xw1AO2UUTPBGKMn6Uox8M/c6EyBoIETIBjtuz5vYfhSNbJ5k9jz/wB6+kdP0x6iSfCUqmWuNccEMw4G1cngngUq1Wj0+nueFsVl3KGYNuQRtOA6zjdPerjP0iXGvgB8L69g5S5cIUjBMeUrmTyTj0zgUyXqCsTsEkMR5gIP1irFgIGApInmnOn+B7Y3EqCT9UKzeGOTMFZmTODEYqfUui/R7TP9W2is5UNGQZBHlM9sSJmJFJ8iGuPRnn1e18wP1pGAQZZXmfxAgVC51AiGFoPMggK2DAI4OR/3pJ9JsjPhgQtsjJJB3EtcOYM7QI9xWvPSN7kbrqK/nH6vbbEDC75MEg8DvNClXYnHwV/pV0GDsDQDAJmckfL0n5VRpesuHWNpMgTtGWJgY7jOeKMPTQA28XGTcQ0kQChOAQRwwYye0cVf0/oGnIEW7rAEQ+51LSN0wFI7EYIz+FNzQsGXa/XkJsACMpZC5CAyoLeU7jElTmO/rSvT9eUTNq27EjzEBjySYAMKTI49BTDqPw9pwq3GNwSTPiXBBBknzJJkGBkCd3elzdCsqSwa4TLwtsNDQBsC3GTBmTmZiMd5i1RUouyX6dBORgehI9e/zP5D0owddtFllGCx5gpGY7gnjil1rpNtmU+BqlthkRgzKC87gWBCkyY3QBA2xwaM1nwzaIBtXygKSVuyWJgkYEZORECPem2hVLwrt69brLaDku7xMYhio2xO2ff3q69aKqqoboK3WTLDYGUBgQu6CSDkgdhQw6NYsy6u5dBuW5tgBhG0wGIZQZ7+nvVVxngqt83LhfeLfhDDbQGJMyx9gBxS/oaXqOafrBDBmBfEfVWYzIEfj91ct6y3tDXm2liwIhNwiCJQNIBJP3VFOgu4At5b1W0wHGSXLwM4iKpf4cIeFuCRJkqJ7cqG5MmPWDVZIlxfh231DcZVkwO4jEmJI++jtN1G3cJVILMCCoEboBI2k/fM8ia907pdhVgfrLrsom4h8MgCSAqklSJA3d4NWfoq2twFLL+IEJDI6IJKlc2mJAHmGe09zTyDAA1fURtDeGNu4LIWDJE4z6dqZWdIjuiWyGZ5C7WVjgDcI3SmZyYmKotdFW23hLdJYz5Qm8YiWY/Zx957RBoC/wBOsBiHeHViCdjKZzAXaZYzmZFLL9hh+hjrOmtbcK6+Y7ZBKjcWPAzk+w5qnSBDLKwTzAjepJgTt8q7oeTxx71JdQ0b2cO0RuYktK+VRxBYQPXjiudTTc7KxVZ/ZAHLDBLFo8ogz3mCIxRk/rBxXxFWtW3uneigeWIM47mR65++h9yQ360g52nY0EweYEKJgfz7r9TodQJQWy0jfysELMsJMng8elDN1K2bZDNbEZK7vPE9xGCar/Sf8Grtahv1sz9QnfiM5Cg8jueKu06qyiXQEYO0l5juxkZ/hFLdJ1Bb6kKUW5EmcqVxicEEe09qo+glsqbZ9STcWT8g4/0KNgq8NFrutCwSNSrq2WCI4uiCSIJQeUwB6R2irtH1u3qBbD6eTe3eECSVOxmA3OACvaRPIIqnoujVEVbywoUC2LN1WmGLBmg+XDDHHI7GvL0nwLtm5obNtlAyLjuR4jSPJAKkwZn1Fc6Z00NrfQdzKLuktAFWLgF8EBgoDgk+hgGTik2u6bespNjwwcK4g/Wgk2x4klTEeXdifatbperW1AQhlYfWAQxuJJOI5Jz6mguu3Gu22VFNthcRiwJ3HyFYgLJbawEe0dqVsKM34d57BfYlt9ha4V2KFAYKP1nAmRgf0j34zuv6XddhbtwzjdKSu4EBGMSZMbgAPf51otNoL6gKysVgjG4NDCCyGRxKnEx7VZb+HB4pNy2+5bY8zl7kDBKsGBxDzMR5uZkU6V2TKLaB+l9FNnS79QqWnuOwU3FD3bZWPOjQduG+6Jq23qbbHxDeG3zpt86kqykyHBIY7gBn50Rpunm6i2pUpaDwGuWECktuYwxUmW+4AVbY+GVLKjpbKAqpEnaC4YAF0B2xzPqAJM1NyvQ8aA7T6VGddyNjebhtXRcDBgEWSYH1jkDPHpUtD1G3dZVA7v4izqI25hlm5Bwcr8uKs0/QN/7BrYeTtsu5S9AlS20gEg8TPE1Vr+jfR2F27aDAxuKlXO4wBt2sTnInsRV2CiEgX9qCzbFsAK+0XtgEzu+uZPrPEn2qtuj6htyvZbw/shm3J3PkMLuJ9ATVq6uz4YWXS65Gw7QBBC7YZzFvvkxxmJqen1H0j/7ibq7lZLtl7hHILlhIACkZHqQMGknobWyVvTapRCpdYQNv7cAEQoHMDBHrAqWts3M2rguMHTYCTdKSVgSCeAeS2IWrbHRbIt+Cvijz+Kdq7BJBAIQmSuZwCRTHR3b1htwvvyWUEXCHLAg71Hocwe+alrwpPRlx0ZGc7yWhYJQqgYqRnd3EZrQPdV0XdqQyhh5blwXEwrRumAGwBE0LcuLbG9DqS5ZXCXDdYFl4O15BUTM8SB3FNbPU9VsJ1eqti1c8RQJBJCwGYEKIywEe3eadslJCBkCFmBVyuGG62WJdQ1vwwoZbiQYc/wBIGJImrdNoLZdA+4sxukk3Ni4VW3SFhWI3Y98Qasu37Soy3Lt1iSUfaWEhFm2SrYckqoBExun1pdrev21ktobfhgxO/DbmBDskzwqEj39qakGNDsXX2MrPvtpad2ndLAMIIZlgk8AY+qYPapaLrg8K3m5Zmdik5AJOzAOCeY96TarqtlbpQMChDZD31VpgklLy7ZgESpOfvhxoOn6cAEXXTdg55wAIMT9UxFP+xf0H6TqNx2VRcub5G2TM5VSwJEfa9fWlq/EVm4V3FrpYnYrKkuUwQSw8pB2+nE9jXfiXpVu0v1nK7lUztYICGjB4Aj171m01tsMG2MzgfX8IMfNAef7UBTPODS0Mb3r1vexNrYXYkqCrsJAIZgkxO7k+9Qu6m0VP6otMEsJR0AEGSIZRPvxJo/puttGCblsnJRUVluTBKswJMQcEbfeaoW/ccvvbxIZkuKyKVndukRJIAIz+VOwoV3uvaVAVs2WfBWfFdVIyAQZM9swMUo0evVLou+GywceGwPHZtw2v3kkZmi9Z1GxvDWU8TzByUTcMkE7gcgcCIirbOit6zVtujToZJhkAA5hROJ82SMdxTM6C7GussCE+mOZO4L4O4EnjiRzgcUde1Vu1cCvacuVBe091bZzAC3FtpLHzLOftCl2n6RZtX0PjpsD4urfUsEOVYImd07cCKfPcsO1wHURbfzbhYuSzliWLgfWWY4JnNJspIGTqVs3VRNMlu6CwUi8d6kgbgQUnMgQezYigdH19WUXWWxp929G3Xbik7YBTaijGB34JNS1WqQNbZbzlrRBCvacEkD6m7lVmCAeI7Uiuolxyb6+TDhFJIBYksA3oJ5p/Ni66HV/r2lULtuaV9sttu+JdQN/ZG5R6crV2l6wNQywmkyYW5bsXFKtHKutwRmJ/Cszb01iGIsjcQ20bSQJPGf7JPFW2LiWiPBtqkEtO23JONhk8Rn/RotDph3T9T+rRijeLthuWNsi4QAGO6AS2Cw7gekMbWqvs7KLl13VpZUVbvmMcgCY9gKzj3H3O1t7gZyCfDuG23GZKkY3QY9a71Dql9rni+bxVVQGLAsSgHm3lvTvFJghxrNBbu3V3LtvqvhjyG05PE3BiDtOf4UQnQkAhrAJ7k6m8T/6TFesaJb2y54v0iFC73QOFPdIOVgyIOTAPFM10sCASBWc+THo0hxqQm+Ftw0q2xqlS4SrKRvdlVfrWwu2W8jMJnmjPiXX6drZOlXUkuNhuG4VRNhBVGBEEkE8GQRFQ0FvqO5Tc1jLt2EjxklhgOFHhzzIwe44p/aR5M3rzIY2ozYB+0xj6xPv91GqFR851r65l8yuQirw5kBZAZiBkweTJ96N0Ou17qwLFWLWmVnZlI2ERuTJOBOJJM0z6p1qwjurBBcTco8RiS20yFYZIHaI4rt+6uosWr2ktIbk+YW1BZG2gMCJBMMcfcapRaJVBn0k27TW0uXA4ELIQLtYRvz5go8w2YnHyria2wVjUI11ioXxLd5gWWQYKgqQBAgCeBWW1fSr+ouFLkvcQBSsqt5RtDS6yC6yVySYJ9qrtvd0ynTpffTuXlouLAO2GBCOx5g5iPvqsf2Dl+jap0q27k+JcUMWMrpzKhgYEs/m9N2c570P1jWPpLSqzC6rFgCttg6kEbASDuBK7juAA3YzM1kG6ldJLNeBJUWdrFyGHkVroLMdtwwTPMtiKFNwHaBdLsgi0QqoXBHm8Vok/WmTJ+VCSByZu+h66zqGOzeriCN4AYjO6CCTAPI9/emPUen+IuxQpfywzGAPacnvXzjpo1FtxdV7Hlbdk7hOdwIiIIBB++tvq/iFy9hNPYZATuufq0CKssGAbHmBUMD3D/gpJXoIt1sV67pl0oDcVfDVIPnncODgDmDxS4aK7bbT3LVvYqb2YzEwiDdn65ndz2p71DV2XUW7gIIY7JJEqykx5G43Rz6Um1Nh7g2kzbVoClCywfKCCGDHG7k1OcY9il2aFdeyXLDG1eYJLuw8MJcDKY2nfjJ4IHFEdU+Iptm4qi0VMqjujXGgyPIPKTjBDSPyrHa4rsljbA8suUO4GGA3kk4gRxBNUWbaSsOQ0qW8qhR3HC+UcZpLkh2N2w3rXV717wgiuRblUXZJWQZ+q2VwMnia7Y6vq1CqdIbhUllZtwPptKjEDFWWNaiXGWOFjczttb6hYAjgjHuZ9KKfrFsZiOMS052kd+f5VS5IsimvoOnXNcpZn0u5WxtZjAn+i07u3B+4il+p1d24HR9OLaPMgMSTP1oLMCOAY9c0fqepDnyhIQxiRAO4gx34+6g7Gq8RtpO3MITndInzfLH50fkV6Q/8ARfduuPDTdtgAQV3dyJbsR/Cnv6WuWwFmwXAkMx2EZJB27iBHY/yrOa18+IWbLIBBGJLYH4flTW3qSSNywc8PiO3Ye9Wtk3Q31nXr1y06XWtlTbQkW0yQZOG7wAx/CkWi1f1j5SRtERI7ZJ7Y4959aL1F1tjYMsIWJk5BweOJz7Vmr90oyxADrldxJQA4Ej3j8Kxknlodj/ql8JBUndt7Erkg7SpXtyD7036N8RKyXX2+c7LjDdtUXCAtwKNpJG5Swg8H2rK9QvbkB4ODGYiMR8ppb0e8Q7LIgpc5BIkAmTnAiciiH7BypjOxeFyUFwqRJ+qoGP7sfv7H2p30npdpCbhvEDzqQF54jcZzlpjjFZDprwzehIwJj1wTmfxp5Y1Z2sOTP1Y8xJ7xz3z91TyOfxjg14aJrXmBBUiQJg88SRPfmqbZJUgDzFVAAGMlzJxj6oz8/as+eoOLiqjj9YUYGJkEAbTPBxn5+0Veti7afYwJUMTChyRggw0HuZ7xAq+Ntqn2OxvqLg3N5l+uw5HrjApabgBiAQXC9+CwBP76sTpzAbkfJ3Fot3AWYldu7y8wDj3oRujOcO6qSQwAt3NxCtz97Aj7iO1a6Fs59N8s7VBieP8A8gX/AKa5f18TECBc9MRcXb/6JNePw6YKl1DHcRIYEhjMKpHmzB+70xVi/DbSZLT5jlHCtuiVK7e8czRaCmVanUMPNbEiI4aMsYjbWn1GmGot6dC0t4VtMiAuPNtPJyO9KrXT7oRbYchBJAFm43JnMmDE+lVaa1dB2pqEMEnaqDyzyTkkZ7YHoBUzpoqOns+kdE6DZ01vapE9zRbaZfUV8vv67UIJN47ZA3EGMmB2qQ6pqIBW7vBnKgxgkEccyK5GvtnQppdGtGohSYJicAiSVBJAkwTAOJqxNeAQWe2bbEgFSGYSispaDAH15+aV810wYrCbdhYmdhUloidvp70dduop2KFHOYCqTiZX7jBE81rySx6Mch/1jRaV7z3WRbhIQuQxKllG1gqAycBaFPUUZTbU7VVYARpiTjaT7/uPyrNJ1OWbO3lWIyT/AHce45zzQfULu1jDSGEwpBAM5k+uCYxBNZPOXbJyNZfu2F1P0i4uy7KecEbSApVgBwAQTIMVV8TQ7i/baA6w/wCrRjuUohbfMkFNmfUP6msbqOpHygkEgwIwDySSYyZxRPSdUxlZkSSAWIAx93H4Zrbjyj2yW70H6e2pXdtO4OrtMLywY+X0lT3ruldU2ZHlG31mVAMY+VVayy4UMxWN6SM/0udxOIz90elBglQGydoyAM4AUn5eU1vVg3TNFpbgATbajeSRuVlACkguJ5Pmj8atfUrJVecSVnYRM5gy3YSM0BotQWtyrES4yQrttyFVhPl+q3z3Ckl7WXCxtohgiDsB3Etmcd8x/OuOfHJyropyHr6kMzLuLOBLEiF5jBPsSPuoe7rA67h2yxkYBIiSDJGM+npSkau9aCpeSBcUMNw8zLnafXmOfShvHjg4zPv8/atIwIbNHcgyXcAv9YBfLcEeaQMkR3PqPlXPpaKhVDECZk/kx+VKtNana7F3YEnygBYG4DzlgRgA4Hc17Q2mYndP1WYcESVJB+WRjtmmuMNjW31Dc68ZVjAkQeTn1gT+FBdV1X20aIPqZzPIP3VXoNPcDTABQPnkEx27RmvavThm2lsYJIyxnk54jH41SgT8K7uoaAJLA8+nvmi+j27j7QqEwxJA7lhCySed0Y/dXNTetooW0i94ZzBWTIJM/ZkZ9qLXW77ZW3b379rsFV7ilgVCydvI3MarEaiUWdDce4VJW0FKSzttQGZyRJBP1QY9aY6LpNsS1+7pHVWLbhq1JgMYUpIMncYOMDMYrl5r11P2BDbD5iqgc+UwTMDHagr42WFTUKpcz512cBsAkfaiRx29qrrQ6rYVq307eIi7ipd2hHt7YYkjaM1V1hLN66rrvUBESCUAhFgY24yP9TQFzrLIP1SoS2TKlyMAAQABgCq06zqG+s1wcf7uFHtCiY++ikFBN7RsAF3oQe4JIInAfaCBHvxUE6Y6XNy3dNO0/wC8wC0jHl82CD99STUOTI1It+x3/wCZTTnpmqYHz9QQj0Kqf8opKPgrS7EWg6adrS9kEtyzlSIPpHB/jV1/ppZf22nJxM3GPY8QpjtWl6t1cWrKXVe3cJu+HIIVYFtmeRBI5WCKz2j+LblzUIj3Et22YBtgHGZlzJ++KKY9UDX+nMf97YEZUKz7hnt5PyB9aqvdId8PqLDAZjxLgUznEpmvo1i/pYBF+2f+cH8zRj37AUEm2cnkr7fj/wB6rBkfkR830uguWyGtanS24/oXI+5gwUN6Gaf9Tu3HZbumFo3PAtW2bx7WGVf1hVS5EbjI+dab6PafIRI9ZWq26Mm6Lb2mLDaQQJElfqzG484x2zScX2UuSL0Y3Q/DVkqGvtda6wHiHdafJ584Yk9szR6ak6Y+FpVuXBmRcJ8vAO3fheeByY+dB6pgeLEDe4kJ5mVWOzA9RE/fS9NejB2csx4+tECJjHA9o7Vx83LKWkaqNMbaWwLlpbd63ds2wWARnjd/fMk7ZMBePwobrl22tspZsW08gJZFAPrIZche0n04oO715xbXw3CWy20Q2eJ86xDcGeJ3DND29ba8RjBBK5IwWmJUJJC7cQRgxWK/Jbcnrwt1VAn6WKAWyp2KVO2ecmZnPcRie9OG16MZD3AOANzjaB2xSm90pnko26CBuaBg5SST3A/KtH8P/D1l0c32lxcIG246CNqkY2GeTmtfxqRCsT6F2VQd4Cjwxljw0YG7MwSPaKH6jqUIDwdxYQCWwq8gHuSSCfl2rR/CvwidQgu+NbWEQlWLC4jEEAt5ZGBgTkGKr6z8GoEN1b9oLKiHchg7Bg25pOFwSvea1k432GLMT1O/JAEADICkkKW8zDdycmojUArzyeIwCeY9q1vTvglGQsLqXSyMLW1iqs8kTPpgx8vwV3umhUZjaW3CW3gs287jsO3bIJDTgkH2xWkaa0S0ILrmP9flRGkuAESdu0Y9TJHJ9BTDS9OAPnSYe2vmY2w0kTyJiPbvWl040rm6yaezbZFh4AuWQScbAwgkANyPSqEkAdHsi94nimV8O48tG0BQ7EmWECNo3epGDWoTo+mtJfe0ttElEVmTxkuKZ/VySCWMAGACD7TWH1vU2tgOjESXgso3Eq+CpK4MifbHtXbXUrpUF9u3yMCzLvkHBZwpLTAwT86uxn0h109i3hUt7grz+r3IbhceVwCQFjAwQDWS1qOx+k2XPjvLeUudjKwVg5jaBKhhuwc+lIm6hdJi2yks0kIq7iSQckLmT60V9KvBmtuqAozby920qEsZMAsA2TBIJqEtlXYNrLm5/wBcyHYCABhAD5icSc8emcRilwe0wm2j7vNiZH9gT6cyTTTU3kY5ti4+dwthDaWR9lkJV+0ZMUd0zTXrllms29Oq7vDKbCGZlBYfUG4MADBbB3n7qJoJ6f8AD2oNveltcR5me0AsrAA3HaD7TMVzqGgZjcueFatqp2r+sQqeBtHhk47yPX3rlix1GMae+yA5PhOV74MCBz3A4+VB6jqVwzaYKPsGUKqpAJ4KqQ0rjEyI70lYaC7ejUJLbioEkWWUPBgEk3uVwMqCRS0jSKQ7i9sJ4NxQ5wJ2sBtH4U3sdE1Lwyaayo7b9yckEkIzFhOeY5NGnSdQQqRprJAHmCFXLe5zuHYYB++nsltCvTracn6C1xQDwSlz/m37ARntTlelalxua8xiSQCTgDMjjtPFVJ8QaseU2bYYbfLIX7QmAx5j1jn2qV3r+puq9pdNbO5mTbvLP5G8w2IQ0BlGTAMd5oFv0c9K6IQoD3LjqOAZG2e3Yjvg0Z+gNP8A1a8z6/upRodf1G3bFsaAbFJKqrKFBMkkIbhyTn8aq1PxZqkIFzSFD3kETx9XEHvx6iqTS+ENN/TQHpqKICgfdUBoF7j8qQ/+KtQxG3SO3eILyR9SCsjEZHNRtfFd/j6IxYYITewB74RCR99GvBYy9NCen2j9mfw/0Kk3R7XJRRNILnxZdH19HfA9StwD8Snv+VVJ8aKWC/R2MxgEs31RuIWOJ3fdFP8Aj4GMh+3SrUfVWOapPRLR+x+R9qU2fjJI8unucTgTkROflntROj+Mbdxtot3JPpyCNsSCR6nzcCKX8AxmGH4cs/0R+X8an+grYUDyiJIEKYBAjt7GqtN1C9cYy2nsAHl7ltiwBIBAUtyB3Ne1vSBcMrrbVxiFzvCKCGHYdtu7PrtouI1CYP1GzZtiblz2AMD/ANh70ItrTKGa29gNtEAuhLEsuBEQe+4+ld1fw2wtuLbszREWygRpPE7piPUduKQ634Z1JkeBdIYEAgWruSSADsMyYPv6VLp6LUWgm5aXeFu31Tglg4XiYEIG/ojI9fWl17patba8GG3xFRh5QY2sQew2nyrEhjPGK2XxBqr96zdtjTazdct2kAbT3iJtsWJMLSLQ6ULbVb+ivswPlIGotGCokbfDM5zPHNR+KK6LybMVo9bsbeEBI+zvnJ5ZVjiFjkxg1d0rpj37zeGoI85JLbUWf7RIjLARntX0j9OD6raJyBCmbZzt4lmsiT++vf8AiGx9rQnB/q1X5R5Bu4/KanAvQB08W7di74ljxbe+wFt7LYJJW4FuDxVYEDY0DAO6jdXY8527SMETatA5E5AWut8TabcSdIwkgv5dswGCyZG4iT8pNU3PibRzjSM3uWg/m1GLKsymj6yqq0yD5ZIJDjYynIPlYSoEe2PSmPT9dccN4Fy2bks83Sr3GBgbURRtXJMnmsF9JO4xwcEeo5+fan3QNZ4L7hGyIYmfrBTtIPp5iMGuaXGqtGYz6v1LU2bi+IrKkqrbT+raPNKjBnEx7Ckl0NuP1rrm6GJQMSysQwcSJBbJzmTR/U+oF7RX6xMgYBbtkHJj1Apn0jUq9y1btW9QBMlnZQFVPOdxVPMMEAGDkZrbh1HYCk9G1Qi41k4cvLsAQAABuHI7nNGdLTwUbc0lzugJccNH2AwCoJzLbsTXm666a0m1cdEIhzJYlcltwbnOIPrV2t051hAs3F2KWEBGXwyYIDwYeYIHYQa0hNvsJJALASBf099l3MYnbhpJIIEYngntzV3V9NbAtPY2WSEDbLlwG6e6nYxK89hk0wX4P1RUL9MxERDwAewFFr8CMQBc1DkYkBRmPnJrbEzc0JdNpdc5VzvUNG4tFpSMYAtiYIq3U6JrSXmu27VxW2qiL4kqJYnaGXk4mK3FnphSBvcgAKATiBx/qasuaERn9/epafwFMwPSejLqAd9z6OgaAgtsJkDzS0D8qar02zpzC+Hc2ncPFBIIEDDKRx6RWqXReh4/jPp8jXfog7/wJp2yclZmbXxDevMRscABUX9ddCALEhAk4IAH8RSd9baXqNtlAUpqLSAm6xhdwViAZ5Ekk+9fQRajgkfeKF/RYmQMzMxSp+DzvsNS+p4u2h/zg/uqzcp/3qEexoQ6Qn5/IV1NGV4JH4fypmYB8YFBp2Y5MqodT5xPeP4UL8AwdMwViIuNO4qXMwfMZE8zmj9d0lbgO/axmSSAZ9ORVOn6PbQQEQf8q/hAFL6Vaqh3sWMsDn17V1rSEGJK9xyMj75paLKWwWKrEj66qF9sgD99U37qL5iUPsDzgHAmO4P31WSJxbMz8RhLWsRELooayHtwNrbmSROPLDEVvktBPKECqDAGYjtgCBWH1GksXLvilQTz5mYwRwNm/wCX404XVsQ8KAQSMNcDD0OWgZkduIrNciN5cMmkPLtu4bTCyVLOAFJZrYAY5ZWzmJjEVhdV0hluC3dsG2GW1suneSFJJZyrbQ8MzHdI+qI7UxvWn8pt3T9Y+ZmIAhUYhQW4IbbPtTjpZ0V57dooviH6qLevFgqPFwksxG2ZgdwOBUvkfhpHjxXYh+DdKwe6l5967h4YYttZpYEmW8zAKOZ9pp7qOq2Sm1LttdzlZ37BCFS+DBIyB6Gafa34WtyxRTv5HiMWtZL7pWCJ4IMT7jNK1+G7qiyPG0xgMUDWpA3wTu8uAO244qozdEy41ldmZ6nqL5G2y8+YOqm4m1lQiQR/RJnucH8CnvA7lRQLqqp2DbImIyfKRyDBNWdd01/6TZNzSWGd5tKuwm1cgwrC3uifNP4VZa/Uu7nptrKDcFN3amMOoG4WxkkBYjFKysUL7X0o3Lha2xUhfDbbaJEbNy8xBG4/d70s6tZv3Wi4FRBfVxdYW0C20EFfQkESBma2fSfoTIrXtPa+kXFmbamWB8qk+aS05MCOPes/1X4cXT2lLhfFVru79pncP1c7xAKgNxEhh6U29Ao77HVrRrAbwkk5wE4ORGQsxHtPtRmgsAAfqwDGYnvMAEGsIt/wjdvFUIO0qp4WA2ARHIUfMye9VaXqG3UaktDKiqyqCVUTds2/KBIn9ZMQeO1NbRL0z6RZd4yLqc4D3OATEw0ScYqHjMVVj4ykjKm7cJHzO7BjMe8VhtLr/wBbqXIYrb8IqBccAB8Hy8T34qOj6s4uaosbm21cCqq3HgBnKAZbPHrRQWbmxqmIkvfUAGVa4xbHsT35jmoW7+4Bt10T2ZjP31jbPXH8S+WuXglt7aqoeRDozCZM/YPel2p+JL4dx49wAMQAdjY7Z2UqQ7MUv8/3GmWh+ov/ADfxr1erFgx58OftB/e/hWz+xc/4Z/eK5Xqh/wDLJXZ8167/ALTd+Z/6a0n/AMOeNR/et/5q9Xq1h0gl0z6Ra4X7q4/I+Zr1erqXRy/SNztVlr+Ner1IZV/vP/1r+966f4V2vU0BIcV5uV+Z/dXK9Wnwllj8UHd5HzH72r1erNgW3/rfjXE5PzrtepMZlOt/7Rc/ur/00Lo/2Q+6vV6sJdnXx9INHKfK3/1GmPwf+31H/Cv/AMa9XqmJrIG+HubX98/4Vut837Bvncr1eqo9kxI6v/L/AJ1rH/Gn/wBLu/K3/nr1ep/BMJ+Of9p0n99v8Na2L/WtfJP+mvV6khfTE3Psf8bWf4wqnr3+y2vlb/whXq9VPoEZTV/sm/u/5HrNj9rqv7o/xrNcr1EOhy7DbXGp/uWaut/X13/Ft/47V6vUySy59XV/8TSf4GopVrv2j/3jXq9SfQ4n/9k=" + }, + { + "type": "image", + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/4b/La_Tour_Eiffel_vue_de_la_Tour_Saint-Jacques%2C_Paris_ao%C3%BBt_2014_%282%29.jpg/2880px-La_Tour_Eiffel_vue_de_la_Tour_Saint-Jacques%2C_Paris_ao%C3%BBt_2014_%282%29.jpg" + }, + { + "type": "image", + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a8/Tour_Eiffel_Wikimedia_Commons.jpg/1024px-Tour_Eiffel_Wikimedia_Commons.jpg", + "text": "The Eiffel Tower is one of the most famous landmarks in Paris." + }, + { + "type": "image", + "url": "https://upload.wikimedia.org/wikipedia/commons/6/64/Dall-e_3_%28jan_%2724%29_artificial_intelligence_icon.png", + "text": "i love machine learning" + } + ], + "compute_scores": true +}' +``` + +## Configuration highlights + +- Base image: `fredbaseten/flash-attn-truss:v0` +- Predict concurrency: **16** diff --git a/nemotron/llama-nemoretriever-colembed-3b-v1/base_image/Dockerfile b/llm/nemotron/llama-nemoretriever-colembed-3b-v1/base_image/Dockerfile similarity index 100% rename from nemotron/llama-nemoretriever-colembed-3b-v1/base_image/Dockerfile rename to llm/nemotron/llama-nemoretriever-colembed-3b-v1/base_image/Dockerfile diff --git a/nemotron/llama-nemoretriever-colembed-3b-v1/config.yaml b/llm/nemotron/llama-nemoretriever-colembed-3b-v1/config.yaml similarity index 100% rename from nemotron/llama-nemoretriever-colembed-3b-v1/config.yaml rename to llm/nemotron/llama-nemoretriever-colembed-3b-v1/config.yaml diff --git a/nemotron/llama-nemoretriever-colembed-3b-v1/model/model.py b/llm/nemotron/llama-nemoretriever-colembed-3b-v1/model/model.py similarity index 100% rename from nemotron/llama-nemoretriever-colembed-3b-v1/model/model.py rename to llm/nemotron/llama-nemoretriever-colembed-3b-v1/model/model.py diff --git a/llm/nemotron/nemotron-3-nano-nvfp4/README.md b/llm/nemotron/nemotron-3-nano-nvfp4/README.md new file mode 100644 index 000000000..6841b4c6a --- /dev/null +++ b/llm/nemotron/nemotron-3-nano-nvfp4/README.md @@ -0,0 +1,59 @@ +# Nemotron 3 Nano + +Deploy [nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4) | +| Task | Text generation | +| Engine | vLLM | +| GPU | B200 | +| Quantization | FP4 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.12.0` +- Predict concurrency: **8** +- System packages: `wget` +- Streaming: **enabled** diff --git a/nemotron/Nemotron-3-Nano-NVFP4/config.yaml b/llm/nemotron/nemotron-3-nano-nvfp4/config.yaml similarity index 93% rename from nemotron/Nemotron-3-Nano-NVFP4/config.yaml rename to llm/nemotron/nemotron-3-nano-nvfp4/config.yaml index 87ffeb8bb..97dc549e8 100644 --- a/nemotron/Nemotron-3-Nano-NVFP4/config.yaml +++ b/llm/nemotron/nemotron-3-nano-nvfp4/config.yaml @@ -1,3 +1,4 @@ +description: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 for text generation" base_image: image: vllm/vllm-openai:v0.12.0 model_metadata: diff --git a/llm/nemotron/nemotron-3-nano/README.md b/llm/nemotron/nemotron-3-nano/README.md new file mode 100644 index 000000000..eb83f9e55 --- /dev/null +++ b/llm/nemotron/nemotron-3-nano/README.md @@ -0,0 +1,58 @@ +# Nemotron 3 Nano + +Deploy [nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.12.0` +- Predict concurrency: **8** +- System packages: `wget` +- Streaming: **enabled** diff --git a/nemotron/Nemotron-3-Nano/config.yaml b/llm/nemotron/nemotron-3-nano/config.yaml similarity index 93% rename from nemotron/Nemotron-3-Nano/config.yaml rename to llm/nemotron/nemotron-3-nano/config.yaml index cb506ed2d..46643d44b 100644 --- a/nemotron/Nemotron-3-Nano/config.yaml +++ b/llm/nemotron/nemotron-3-nano/config.yaml @@ -1,3 +1,4 @@ +description: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 for text generation" base_image: image: vllm/vllm-openai:v0.12.0 model_metadata: diff --git a/llm/nemotron/nemotron-nano-12b-v2-vl-bf16/README.md b/llm/nemotron/nemotron-nano-12b-v2-vl-bf16/README.md new file mode 100644 index 000000000..26fae6c31 --- /dev/null +++ b/llm/nemotron/nemotron-nano-12b-v2-vl-bf16/README.md @@ -0,0 +1,57 @@ +# NVIDIA Nemotron Nano 12B v2 VL BF16 + +Deploy [nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:1 | +| OpenAI compatible | Yes | +| Python | py312 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.11.0` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **16** +- Streaming: **enabled** diff --git a/nemotron/Nemotron-Nano-12B-v2-VL-BF16/config.yaml b/llm/nemotron/nemotron-nano-12b-v2-vl-bf16/config.yaml similarity index 88% rename from nemotron/Nemotron-Nano-12B-v2-VL-BF16/config.yaml rename to llm/nemotron/nemotron-nano-12b-v2-vl-bf16/config.yaml index 77dafd856..af53d2bcc 100644 --- a/nemotron/Nemotron-Nano-12B-v2-VL-BF16/config.yaml +++ b/llm/nemotron/nemotron-nano-12b-v2-vl-bf16/config.yaml @@ -1,3 +1,4 @@ +description: "Nemotron Nano 12B v2 VL for vision-language tasks" base_image: image: vllm/vllm-openai:v0.11.0 model_metadata: @@ -17,11 +18,11 @@ model_metadata: model_name: NVIDIA Nemotron Nano 12B v2 VL BF16 requirements: - transformers>=4.55.0 - - accelerate - - timm - - einops - - open-clip-torch - - pillow + - accelerate==1.2.1 + - timm==1.0.12 + - einops==0.8.0 + - open-clip-torch==2.29.0 + - pillow==10.4.0 python_version: py312 model_cache: - repo_id: nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16 diff --git a/llm/nemotron/nemotron-ultra-253b/README.md b/llm/nemotron/nemotron-ultra-253b/README.md new file mode 100644 index 000000000..cbcde2375 --- /dev/null +++ b/llm/nemotron/nemotron-ultra-253b/README.md @@ -0,0 +1,58 @@ +# Briton-nemotron-253b-tp8-fp8 + +Deploy [michaelfeil/nemotron-251b-ultra-v2-tp8-fp8-tllm](https://huggingface.co/michaelfeil/nemotron-251b-ultra-v2-tp8-fp8-tllm) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [michaelfeil/nemotron-251b-ultra-v2-tp8-fp8-tllm](https://huggingface.co/michaelfeil/nemotron-251b-ultra-v2-tp8-fp8-tllm) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:8 | +| Quantization | FP8 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="michaelfeil/nemotron-251b-ultra-v2-tp8-fp8-tllm", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "michaelfeil/nemotron-251b-ultra-v2-tp8-fp8-tllm", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Tensor parallelism: **8** GPUs +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **65,536** +- Streaming: **enabled** diff --git a/nemotron/nemotron-ultra-253b/config.yaml b/llm/nemotron/nemotron-ultra-253b/config.yaml similarity index 91% rename from nemotron/nemotron-ultra-253b/config.yaml rename to llm/nemotron/nemotron-ultra-253b/config.yaml index c4a35d1a0..5b1a5dd81 100644 --- a/nemotron/nemotron-ultra-253b/config.yaml +++ b/llm/nemotron/nemotron-ultra-253b/config.yaml @@ -1,3 +1,4 @@ +description: "michaelfeil/nemotron-251b-ultra-v2-tp8-fp8-tllm for text generation" model_metadata: example_model_input: max_tokens: 512 diff --git a/llm/nsql/README.md b/llm/nsql/README.md new file mode 100644 index 000000000..67614f44e --- /dev/null +++ b/llm/nsql/README.md @@ -0,0 +1,29 @@ +# NSQL 350M + +NSQL is an open-source text-to-SQL AI model developed by Numbers Station. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/nsql/config.yaml b/llm/nsql/config.yaml similarity index 73% rename from nsql/config.yaml rename to llm/nsql/config.yaml index e3863ecfe..85a08f956 100644 --- a/nsql/config.yaml +++ b/llm/nsql/config.yaml @@ -2,14 +2,16 @@ description: NSQL is an open-source text-to-SQL AI model developed by Numbers St environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "NumbersStation/nsql-350M" avatar_url: https://aeiljuispo.cloudimg.io/v7/https://cdn-uploads.huggingface.co/production/uploads/649c7ee8f97bd6fd710a9eb5/nBg1Fyo22RrqRJrkz9IYB.png cover_image_url: https://global-uploads.webflow.com/6348b2d49808811e3f7a0fff/640690727b722a05771960ec_graphic-data-p-800.png tags: - code-generation + example_model_input: {"schema": "CREATE TABLE stadium (stadium_id number, location text, name text, capacity number)", "query": "What is the total capacity of all stadiums?"} model_name: NSQL 350M python_version: py39 requirements: -- torch +- torch==2.1.0 - transformers>=4.29.0 resources: accelerator: A10G diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/model/__init__.py b/llm/nsql/model/__init__.py similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/large_context/model/__init__.py rename to llm/nsql/model/__init__.py diff --git a/nsql/model/model.py b/llm/nsql/model/model.py similarity index 100% rename from nsql/model/model.py rename to llm/nsql/model/model.py diff --git a/llm/openai/gpt-oss-120b/README.md b/llm/openai/gpt-oss-120b/README.md new file mode 100644 index 000000000..8de0d681b --- /dev/null +++ b/llm/openai/gpt-oss-120b/README.md @@ -0,0 +1,54 @@ +# GPT OSS 120B + +Deploy [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="openai/gpt-oss-120b", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "openai/gpt-oss-120b", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Model cache: **volume-mounted** for fast cold starts +- Streaming: **enabled** diff --git a/openai/gpt-oss-120b/config.yaml b/llm/openai/gpt-oss-120b/config.yaml similarity index 97% rename from openai/gpt-oss-120b/config.yaml rename to llm/openai/gpt-oss-120b/config.yaml index c76fdaf12..b77dfc192 100644 --- a/openai/gpt-oss-120b/config.yaml +++ b/llm/openai/gpt-oss-120b/config.yaml @@ -1,3 +1,4 @@ +description: "openai/gpt-oss-120b for text generation" model_name: GPT OSS 120B build_commands: - python -c 'from openai_harmony import load_harmony_encoding; load_harmony_encoding("HarmonyGptOss")' diff --git a/llm/openai/gpt-oss-20b/README.md b/llm/openai/gpt-oss-20b/README.md new file mode 100644 index 000000000..88ea7ab1c --- /dev/null +++ b/llm/openai/gpt-oss-20b/README.md @@ -0,0 +1,54 @@ +# GPT OSS 20B + +Deploy [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="openai/gpt-oss-20b", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "openai/gpt-oss-20b", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Model cache: **volume-mounted** for fast cold starts +- Streaming: **enabled** diff --git a/openai/gpt-oss-20b/config.yaml b/llm/openai/gpt-oss-20b/config.yaml similarity index 97% rename from openai/gpt-oss-20b/config.yaml rename to llm/openai/gpt-oss-20b/config.yaml index a6d4edbc0..cc06a36ad 100644 --- a/openai/gpt-oss-20b/config.yaml +++ b/llm/openai/gpt-oss-20b/config.yaml @@ -1,3 +1,4 @@ +description: "openai/gpt-oss-20b for text generation" model_name: GPT OSS 20B build_commands: - python -c 'from openai_harmony import load_harmony_encoding; load_harmony_encoding("HarmonyGptOss")' diff --git a/personaplex-7b-v1/Dockerfile b/llm/personaplex-7b-v1/Dockerfile similarity index 100% rename from personaplex-7b-v1/Dockerfile rename to llm/personaplex-7b-v1/Dockerfile diff --git a/llm/personaplex-7b-v1/README.md b/llm/personaplex-7b-v1/README.md new file mode 100644 index 000000000..4f335ca9b --- /dev/null +++ b/llm/personaplex-7b-v1/README.md @@ -0,0 +1,31 @@ +# Personaplex 7V H100 + +Speech-to-speech model powered by NVIDIA Personaplex + +| Property | Value | +|----------|-------| +| Model | [nvidia/personaplex-7b-v1](https://huggingface.co/nvidia/personaplex-7b-v1) | +| Task | Text generation | +| Engine | Docker Server | +| GPU | H100 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/api/chat \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `basetenservice/personaplex-7v:fork` diff --git a/personaplex-7b-v1/client.py b/llm/personaplex-7b-v1/client.py similarity index 100% rename from personaplex-7b-v1/client.py rename to llm/personaplex-7b-v1/client.py diff --git a/personaplex-7b-v1/config.yaml b/llm/personaplex-7b-v1/config.yaml similarity index 90% rename from personaplex-7b-v1/config.yaml rename to llm/personaplex-7b-v1/config.yaml index b4b778edf..544001ed0 100644 --- a/personaplex-7b-v1/config.yaml +++ b/llm/personaplex-7b-v1/config.yaml @@ -5,6 +5,7 @@ model_metadata: repo_id: nvidia/personaplex-7b-v1 tags: - speech-to-speech + example_model_input: {"message": "Hello, how are you today?"} docker_server: start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python3 -m moshi.server --host 0.0.0.0 --port 8998" readiness_endpoint: / diff --git a/personaplex-7b-v1/requirements-client.txt b/llm/personaplex-7b-v1/requirements-client.txt similarity index 100% rename from personaplex-7b-v1/requirements-client.txt rename to llm/personaplex-7b-v1/requirements-client.txt diff --git a/llm/phi/phi-3-mini-128k-instruct/README.md b/llm/phi/phi-3-mini-128k-instruct/README.md new file mode 100644 index 000000000..f0ca30149 --- /dev/null +++ b/llm/phi/phi-3-mini-128k-instruct/README.md @@ -0,0 +1,29 @@ +# Phi-3-Mini-128K-Instruct + +Deploy Phi-3-Mini-128K-Instruct for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/llm/phi/phi-3-mini-128k-instruct/config.yaml b/llm/phi/phi-3-mini-128k-instruct/config.yaml new file mode 100644 index 000000000..918ffab65 --- /dev/null +++ b/llm/phi/phi-3-mini-128k-instruct/config.yaml @@ -0,0 +1,18 @@ +description: "Phi-3-Mini-128K-Instruct for text generation" +environment_variables: {} +external_package_dirs: [] +model_metadata: + repo_id: "microsoft/Phi-3-mini-128k-instruct" + example_model_input: {"messages": [{"role": "user", "content": "What is the meaning of life?"}]} +model_name: Phi-3-Mini-128K-Instruct +python_version: py39 +requirements: + - accelerate==0.30.1 + - einops==0.8.0 + - transformers==4.40.1 + - torch==2.3.0 +resources: + accelerator: T4 + use_gpu: true +secrets: {} +system_packages: [] diff --git a/trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/model/__init__.py b/llm/phi/phi-3-mini-128k-instruct/model/__init__.py similarity index 100% rename from trt-llm-engine-builder-templates/llama-3_1-8b-instruct/low_ttft/model/__init__.py rename to llm/phi/phi-3-mini-128k-instruct/model/__init__.py diff --git a/phi/phi-3-mini-128k-instruct/model/model.py b/llm/phi/phi-3-mini-128k-instruct/model/model.py similarity index 100% rename from phi/phi-3-mini-128k-instruct/model/model.py rename to llm/phi/phi-3-mini-128k-instruct/model/model.py diff --git a/llm/phi/phi-3-mini-4k-instruct/README.md b/llm/phi/phi-3-mini-4k-instruct/README.md new file mode 100644 index 000000000..b40a43542 --- /dev/null +++ b/llm/phi/phi-3-mini-4k-instruct/README.md @@ -0,0 +1,29 @@ +# Phi-3-Mini-4K-Instruct + +Deploy Phi-3-Mini-4K-Instruct for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | T4 | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/llm/phi/phi-3-mini-4k-instruct/config.yaml b/llm/phi/phi-3-mini-4k-instruct/config.yaml new file mode 100644 index 000000000..9bb07a3fd --- /dev/null +++ b/llm/phi/phi-3-mini-4k-instruct/config.yaml @@ -0,0 +1,18 @@ +description: "Phi-3-Mini-4K-Instruct for text generation" +environment_variables: {} +external_package_dirs: [] +model_metadata: + repo_id: "microsoft/Phi-3-mini-4k-instruct" + example_model_input: {"messages": [{"role": "user", "content": "What is the meaning of life?"}]} +model_name: Phi-3-Mini-4K-Instruct +python_version: py39 +requirements: + - accelerate==0.30.1 + - einops==0.8.0 + - transformers==4.40.1 + - torch==2.3.0 +resources: + accelerator: T4 + use_gpu: true +secrets: {} +system_packages: [] diff --git a/ultravox/model/__init__.py b/llm/phi/phi-3-mini-4k-instruct/model/__init__.py similarity index 100% rename from ultravox/model/__init__.py rename to llm/phi/phi-3-mini-4k-instruct/model/__init__.py diff --git a/phi/phi-3-mini-4k-instruct/model/model.py b/llm/phi/phi-3-mini-4k-instruct/model/model.py similarity index 100% rename from phi/phi-3-mini-4k-instruct/model/model.py rename to llm/phi/phi-3-mini-4k-instruct/model/model.py diff --git a/llm/phi/phi-3.5-mini/README.md b/llm/phi/phi-3.5-mini/README.md new file mode 100644 index 000000000..2f6bbb085 --- /dev/null +++ b/llm/phi/phi-3.5-mini/README.md @@ -0,0 +1,39 @@ +# Phi 3.5 Mini Instruct VLLM openai compatible + +Deploy [microsoft/Phi-3.5-mini-instruct](https://huggingface.co/microsoft/Phi-3.5-mini-instruct) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [microsoft/Phi-3.5-mini-instruct](https://huggingface.co/microsoft/Phi-3.5-mini-instruct) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "user", + "content": "what is the meaning of life" + } + ] +}' +``` + +## Configuration highlights + +- Predict concurrency: **128** diff --git a/phi/phi-3.5-mini/config.yaml b/llm/phi/phi-3.5-mini/config.yaml similarity index 87% rename from phi/phi-3.5-mini/config.yaml rename to llm/phi/phi-3.5-mini/config.yaml index 9beacb1f1..68f2b7062 100644 --- a/phi/phi-3.5-mini/config.yaml +++ b/llm/phi/phi-3.5-mini/config.yaml @@ -1,3 +1,4 @@ +description: "microsoft/Phi-3.5-mini-instruct for text generation" model_name: "Phi 3.5 Mini Instruct VLLM openai compatible" python_version: py311 model_metadata: diff --git a/vllm/model/__init__.py b/llm/phi/phi-3.5-mini/model/__init__.py similarity index 100% rename from vllm/model/__init__.py rename to llm/phi/phi-3.5-mini/model/__init__.py diff --git a/vllm/model/helper.py b/llm/phi/phi-3.5-mini/model/helper.py similarity index 100% rename from vllm/model/helper.py rename to llm/phi/phi-3.5-mini/model/helper.py diff --git a/vllm/model/model.py b/llm/phi/phi-3.5-mini/model/model.py similarity index 100% rename from vllm/model/model.py rename to llm/phi/phi-3.5-mini/model/model.py diff --git a/qwen/qwen-7b-chat/README.md b/llm/qwen/_archive/qwen-7b-chat/README.md similarity index 100% rename from qwen/qwen-7b-chat/README.md rename to llm/qwen/_archive/qwen-7b-chat/README.md diff --git a/qwen/qwen-7b-chat/config.yaml b/llm/qwen/_archive/qwen-7b-chat/config.yaml similarity index 100% rename from qwen/qwen-7b-chat/config.yaml rename to llm/qwen/_archive/qwen-7b-chat/config.yaml diff --git a/whisper/faster-whisper-small/model/__init__.py b/llm/qwen/_archive/qwen-7b-chat/model/__init__.py similarity index 100% rename from whisper/faster-whisper-small/model/__init__.py rename to llm/qwen/_archive/qwen-7b-chat/model/__init__.py diff --git a/qwen/qwen-7b-chat/model/model.py b/llm/qwen/_archive/qwen-7b-chat/model/model.py similarity index 100% rename from qwen/qwen-7b-chat/model/model.py rename to llm/qwen/_archive/qwen-7b-chat/model/model.py diff --git a/llm/qwen/engine-qwen-2-5-14b-coder-instruct/README.md b/llm/qwen/engine-qwen-2-5-14b-coder-instruct/README.md new file mode 100644 index 000000000..bae332e8a --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-14b-coder-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen Coder 2.5 14B Instruct + +Deploy [Qwen/Qwen2.5-Coder-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Coder-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Coder-14B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Coder-14B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-14b-coder-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-14b-coder-instruct/config.yaml similarity index 94% rename from qwen/engine-qwen-2-5-14b-coder-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-14b-coder-instruct/config.yaml index 17395f233..09ffacb03 100644 --- a/qwen/engine-qwen-2-5-14b-coder-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-14b-coder-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen 2.5 Coder 14B Instruct for code generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-14b-instruct/README.md b/llm/qwen/engine-qwen-2-5-14b-instruct/README.md new file mode 100644 index 000000000..cc997098a --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-14b-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen 2.5 14B Instruct + +Deploy [Qwen/Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-14B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-14B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-14b-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-14b-instruct/config.yaml similarity index 94% rename from qwen/engine-qwen-2-5-14b-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-14b-instruct/config.yaml index 1bc50b8a6..fab818e2d 100644 --- a/qwen/engine-qwen-2-5-14b-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-14b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen2.5-14B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-32b-coder-instruct/README.md b/llm/qwen/engine-qwen-2-5-32b-coder-instruct/README.md new file mode 100644 index 000000000..343b1ab34 --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-32b-coder-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen Coder 2.5 32B Instruct + +Deploy [Qwen/Qwen2.5-Coder-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Coder-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Coder-32B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Coder-32B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-32b-coder-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-32b-coder-instruct/config.yaml similarity index 94% rename from qwen/engine-qwen-2-5-32b-coder-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-32b-coder-instruct/config.yaml index 39f50a264..408ae3d02 100644 --- a/qwen/engine-qwen-2-5-32b-coder-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-32b-coder-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen 2.5 Coder 32B Instruct for code generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-32b-instruct/README.md b/llm/qwen/engine-qwen-2-5-32b-instruct/README.md new file mode 100644 index 000000000..10a592d7c --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-32b-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen 2.5 32B Instruct + +Deploy [Qwen/Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-32B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-32B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-32b-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-32b-instruct/config.yaml similarity index 95% rename from qwen/engine-qwen-2-5-32b-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-32b-instruct/config.yaml index 6536b7dec..ff926bcb9 100644 --- a/qwen/engine-qwen-2-5-32b-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-32b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen2.5-32B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-3b-instruct/README.md b/llm/qwen/engine-qwen-2-5-3b-instruct/README.md new file mode 100644 index 000000000..dde39acc9 --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-3b-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen 2.5 3B Instruct + +Deploy [Qwen/Qwen2.5-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | A10G | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-3b-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-3b-instruct/config.yaml similarity index 95% rename from qwen/engine-qwen-2-5-3b-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-3b-instruct/config.yaml index 5480b3954..c5a0ffc2a 100644 --- a/qwen/engine-qwen-2-5-3b-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-3b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen2.5-3B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-72b-instruct/README.md b/llm/qwen/engine-qwen-2-5-72b-instruct/README.md new file mode 100644 index 000000000..de37f4f18 --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-72b-instruct/README.md @@ -0,0 +1,62 @@ +# Qwen 2.5 72B Instruct + +Deploy [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-72B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-72B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Tensor parallelism: **2** GPUs +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-72b-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-72b-instruct/config.yaml similarity index 95% rename from qwen/engine-qwen-2-5-72b-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-72b-instruct/config.yaml index a69157bd7..bec27fb19 100644 --- a/qwen/engine-qwen-2-5-72b-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-72b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen2.5-72B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-72b-math-instruct/README.md b/llm/qwen/engine-qwen-2-5-72b-math-instruct/README.md new file mode 100644 index 000000000..873bd2455 --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-72b-math-instruct/README.md @@ -0,0 +1,62 @@ +# Qwen Math 2.5 72B Instruct + +Deploy [Qwen/Qwen2.5-Math-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-72B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Math-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-72B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Math-72B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Math-72B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Tensor parallelism: **2** GPUs +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-72b-math-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-72b-math-instruct/config.yaml similarity index 94% rename from qwen/engine-qwen-2-5-72b-math-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-72b-math-instruct/config.yaml index 9d7d59d95..0f5dc2130 100644 --- a/qwen/engine-qwen-2-5-72b-math-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-72b-math-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen 2.5 Math 72B Instruct for mathematical reasoning" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-7b-coder-instruct/README.md b/llm/qwen/engine-qwen-2-5-7b-coder-instruct/README.md new file mode 100644 index 000000000..b42074182 --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-7b-coder-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen Coder 2.5 7B Instruct + +Deploy [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Coder-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-7b-coder-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-7b-coder-instruct/config.yaml similarity index 95% rename from qwen/engine-qwen-2-5-7b-coder-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-7b-coder-instruct/config.yaml index 19402b446..4b59cd31d 100644 --- a/qwen/engine-qwen-2-5-7b-coder-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-7b-coder-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen 2.5 Coder 7B Instruct for code generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-7b-instruct/README.md b/llm/qwen/engine-qwen-2-5-7b-instruct/README.md new file mode 100644 index 000000000..95718dcde --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-7b-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen 2.5 7B Instruct + +Deploy [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-7b-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-7b-instruct/config.yaml similarity index 95% rename from qwen/engine-qwen-2-5-7b-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-7b-instruct/config.yaml index 08205932c..c465bff47 100644 --- a/qwen/engine-qwen-2-5-7b-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-7b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen2.5-7B-Instruct for text generation" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-2-5-7b-math-instruct/README.md b/llm/qwen/engine-qwen-2-5-7b-math-instruct/README.md new file mode 100644 index 000000000..abd86e80c --- /dev/null +++ b/llm/qwen/engine-qwen-2-5-7b-math-instruct/README.md @@ -0,0 +1,61 @@ +# Qwen Math 2.5 7B Instruct + +Deploy [Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Math-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Math-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_paged_context_fmha** +- Plugin: **paged_kv_cache** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-2-5-7b-math-instruct/config.yaml b/llm/qwen/engine-qwen-2-5-7b-math-instruct/config.yaml similarity index 94% rename from qwen/engine-qwen-2-5-7b-math-instruct/config.yaml rename to llm/qwen/engine-qwen-2-5-7b-math-instruct/config.yaml index aa06ec39c..697d2362d 100644 --- a/qwen/engine-qwen-2-5-7b-math-instruct/config.yaml +++ b/llm/qwen/engine-qwen-2-5-7b-math-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen 2.5 Math 7B Instruct for mathematical reasoning" build_commands: [] environment_variables: {} external_package_dirs: [] diff --git a/llm/qwen/engine-qwen-3-06b/README.md b/llm/qwen/engine-qwen-3-06b/README.md new file mode 100644 index 000000000..d7e0d5a34 --- /dev/null +++ b/llm/qwen/engine-qwen-3-06b/README.md @@ -0,0 +1,59 @@ +# library-model-qwen3-06b-engine + +Deploy [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-0.6B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-3-06b/config.yaml b/llm/qwen/engine-qwen-3-06b/config.yaml similarity index 95% rename from qwen/engine-qwen-3-06b/config.yaml rename to llm/qwen/engine-qwen-3-06b/config.yaml index f17889c21..8da3fe294 100644 --- a/qwen/engine-qwen-3-06b/config.yaml +++ b/llm/qwen/engine-qwen-3-06b/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-0.6B for text generation" model_metadata: example_model_input: max_tokens: 512 diff --git a/llm/qwen/engine-qwen-3-30b-a3b-instruct-2507/README.md b/llm/qwen/engine-qwen-3-30b-a3b-instruct-2507/README.md new file mode 100644 index 000000000..49a09fb3e --- /dev/null +++ b/llm/qwen/engine-qwen-3-30b-a3b-instruct-2507/README.md @@ -0,0 +1,62 @@ +# qwen3-30b-a3b-instruct-2507-fp8_kv + +Deploy [Qwen/Qwen3-30B-A3B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-30B-A3B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-30B-A3B-Instruct-2507", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-30B-A3B-Instruct-2507", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-3-30b-a3b-instruct-2507/config.yaml b/llm/qwen/engine-qwen-3-30b-a3b-instruct-2507/config.yaml similarity index 92% rename from qwen/engine-qwen-3-30b-a3b-instruct-2507/config.yaml rename to llm/qwen/engine-qwen-3-30b-a3b-instruct-2507/config.yaml index f014a7f84..ece2a67c2 100644 --- a/qwen/engine-qwen-3-30b-a3b-instruct-2507/config.yaml +++ b/llm/qwen/engine-qwen-3-30b-a3b-instruct-2507/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-30B-A3B-Instruct-2507 for text generation" model_metadata: example_model_input: max_tokens: 512 diff --git a/llm/qwen/engine-qwen-3-32b/README.md b/llm/qwen/engine-qwen-3-32b/README.md new file mode 100644 index 000000000..3f9eea508 --- /dev/null +++ b/llm/qwen/engine-qwen-3-32b/README.md @@ -0,0 +1,59 @@ +# library-model-qwen3-32B-engine + +Deploy [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-3-32b/config.yaml b/llm/qwen/engine-qwen-3-32b/config.yaml similarity index 95% rename from qwen/engine-qwen-3-32b/config.yaml rename to llm/qwen/engine-qwen-3-32b/config.yaml index 675f6ddbd..150c0bc5c 100644 --- a/qwen/engine-qwen-3-32b/config.yaml +++ b/llm/qwen/engine-qwen-3-32b/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-32B for text generation" model_metadata: example_model_input: max_tokens: 512 diff --git a/llm/qwen/engine-qwen-3-4b/README.md b/llm/qwen/engine-qwen-3-4b/README.md new file mode 100644 index 000000000..9994df61a --- /dev/null +++ b/llm/qwen/engine-qwen-3-4b/README.md @@ -0,0 +1,59 @@ +# library-model-qwen3-4b-engine + +Deploy [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-4B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-4B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/qwen/engine-qwen-3-4b/config.yaml b/llm/qwen/engine-qwen-3-4b/config.yaml similarity index 95% rename from qwen/engine-qwen-3-4b/config.yaml rename to llm/qwen/engine-qwen-3-4b/config.yaml index a20f2c1db..cc2d3dfd1 100644 --- a/qwen/engine-qwen-3-4b/config.yaml +++ b/llm/qwen/engine-qwen-3-4b/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-4B for text generation" model_metadata: example_model_input: max_tokens: 512 diff --git a/qwen/model_auto.py b/llm/qwen/model_auto.py similarity index 100% rename from qwen/model_auto.py rename to llm/qwen/model_auto.py diff --git a/llm/qwen/qwen-3-235B-A22B-instruct-2507-trt/README.md b/llm/qwen/qwen-3-235B-A22B-instruct-2507-trt/README.md new file mode 100644 index 000000000..5b4c57cb7 --- /dev/null +++ b/llm/qwen/qwen-3-235B-A22B-instruct-2507-trt/README.md @@ -0,0 +1,53 @@ +# Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 + +Deploy [Qwen/Qwen3-235B-A22B-Instruct-2507-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-235B-A22B-Instruct-2507-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:8 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Model cache: **volume-mounted** for fast cold starts diff --git a/qwen/qwen-3-235B-A22B-instruct-2507-trt/config.yaml b/llm/qwen/qwen-3-235B-A22B-instruct-2507-trt/config.yaml similarity index 95% rename from qwen/qwen-3-235B-A22B-instruct-2507-trt/config.yaml rename to llm/qwen/qwen-3-235B-A22B-instruct-2507-trt/config.yaml index ef161a348..9cae82ce2 100644 --- a/qwen/qwen-3-235B-A22B-instruct-2507-trt/config.yaml +++ b/llm/qwen/qwen-3-235B-A22B-instruct-2507-trt/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 for text generation" model_metadata: example_model_input: # Loads sample request into Baseten playground messages: diff --git a/llm/qwen/qwen-3-235B-sglang/README.md b/llm/qwen/qwen-3-235B-sglang/README.md new file mode 100644 index 000000000..881b1d58a --- /dev/null +++ b/llm/qwen/qwen-3-235B-sglang/README.md @@ -0,0 +1,56 @@ +# Qwen 3 235B SGLang + +Deploy [Qwen/Qwen3-235B-A22B-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-235B-A22B-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:4 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-235B-A22B-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-235B-A22B-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.6.post1-cu124` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/qwen/qwen-3-235B-sglang/config.yaml b/llm/qwen/qwen-3-235B-sglang/config.yaml similarity index 94% rename from qwen/qwen-3-235B-sglang/config.yaml rename to llm/qwen/qwen-3-235B-sglang/config.yaml index b78764125..2a8825c22 100644 --- a/qwen/qwen-3-235B-sglang/config.yaml +++ b/llm/qwen/qwen-3-235B-sglang/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-235B-A22B-FP8 for text generation" model_metadata: example_model_input: # Loads sample request into Baseten playground messages: diff --git a/llm/qwen/qwen-3-30B-A3-coder/README.md b/llm/qwen/qwen-3-30B-A3-coder/README.md new file mode 100644 index 000000000..bfe7202b5 --- /dev/null +++ b/llm/qwen/qwen-3-30B-A3-coder/README.md @@ -0,0 +1,56 @@ +# Qwen 3 Coder + +Deploy [Qwen/Qwen3-Coder-30B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-Coder-30B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:1 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-Coder-30B-A3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.10.post2-cu126` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/qwen/qwen-3-30B-A3-coder/config.yaml b/llm/qwen/qwen-3-30B-A3-coder/config.yaml similarity index 94% rename from qwen/qwen-3-30B-A3-coder/config.yaml rename to llm/qwen/qwen-3-30B-A3-coder/config.yaml index b6c5c228a..3cc4f7305 100644 --- a/qwen/qwen-3-30B-A3-coder/config.yaml +++ b/llm/qwen/qwen-3-30B-A3-coder/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen3 Coder 30B A3B Instruct for code generation" model_metadata: example_model_input: # Loads sample request into Baseten playground messages: diff --git a/llm/qwen/qwen-3-30B-A3-sglang/README.md b/llm/qwen/qwen-3-30B-A3-sglang/README.md new file mode 100644 index 000000000..eae0813fa --- /dev/null +++ b/llm/qwen/qwen-3-30B-A3-sglang/README.md @@ -0,0 +1,57 @@ +# Qwen 3 30B-A3 SGLang + +Deploy [Qwen/Qwen3-30B-A3B-FP8](https://huggingface.co/Qwen/Qwen3-30B-A3B-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-30B-A3B-FP8](https://huggingface.co/Qwen/Qwen3-30B-A3B-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:1 | +| OpenAI compatible | Yes | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-30B-A3B-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-30B-A3B-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.6.post1-cu124` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** diff --git a/qwen/qwen-3-30B-A3-sglang/config.yaml b/llm/qwen/qwen-3-30B-A3-sglang/config.yaml similarity index 95% rename from qwen/qwen-3-30B-A3-sglang/config.yaml rename to llm/qwen/qwen-3-30B-A3-sglang/config.yaml index 05cc6157c..70438bac2 100644 --- a/qwen/qwen-3-30B-A3-sglang/config.yaml +++ b/llm/qwen/qwen-3-30B-A3-sglang/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-30B-A3B-FP8 for text generation" model_metadata: example_model_input: # Loads sample request into Baseten playground messages: diff --git a/llm/qwen/qwen-3-30B-A3-vllm/README.md b/llm/qwen/qwen-3-30B-A3-vllm/README.md new file mode 100644 index 000000000..b5b637cf8 --- /dev/null +++ b/llm/qwen/qwen-3-30B-A3-vllm/README.md @@ -0,0 +1,55 @@ +# Qwen 3 30B-A3B vLLM + +Deploy [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:1 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-30B-A3B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-30B-A3B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.8.5` +- Predict concurrency: **32** +- Environment variables: `VLLM_LOGGING_LEVEL` diff --git a/qwen/qwen-3-30B-A3-vllm/config.yaml b/llm/qwen/qwen-3-30B-A3-vllm/config.yaml similarity index 93% rename from qwen/qwen-3-30B-A3-vllm/config.yaml rename to llm/qwen/qwen-3-30B-A3-vllm/config.yaml index e7f64104b..c0e5db27d 100644 --- a/qwen/qwen-3-30B-A3-vllm/config.yaml +++ b/llm/qwen/qwen-3-30B-A3-vllm/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-30B-A3B for text generation" base_image: image: vllm/vllm-openai:v0.8.5 docker_server: diff --git a/llm/qwen/qwen-3-32B-sglang/README.md b/llm/qwen/qwen-3-32B-sglang/README.md new file mode 100644 index 000000000..5b5e59aa6 --- /dev/null +++ b/llm/qwen/qwen-3-32B-sglang/README.md @@ -0,0 +1,56 @@ +# Qwen 3 32B SGLang + +Deploy [Qwen/Qwen3-32B-FP8](https://huggingface.co/Qwen/Qwen3-32B-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-32B-FP8](https://huggingface.co/Qwen/Qwen3-32B-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:1 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-32B-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-32B-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.6.post1-cu124` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/qwen/qwen-3-32B-sglang/config.yaml b/llm/qwen/qwen-3-32B-sglang/config.yaml similarity index 95% rename from qwen/qwen-3-32B-sglang/config.yaml rename to llm/qwen/qwen-3-32B-sglang/config.yaml index 8e1e22488..a4d88d4be 100644 --- a/qwen/qwen-3-32B-sglang/config.yaml +++ b/llm/qwen/qwen-3-32B-sglang/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-32B-FP8 for text generation" model_metadata: example_model_input: # Loads sample request into Baseten playground messages: diff --git a/llm/qwen/qwen-3-next-80B-A3-instruct-sglang/README.md b/llm/qwen/qwen-3-next-80B-A3-instruct-sglang/README.md new file mode 100644 index 000000000..657509ec5 --- /dev/null +++ b/llm/qwen/qwen-3-next-80B-A3-instruct-sglang/README.md @@ -0,0 +1,56 @@ +# Qwen3-Next-80B-A3B-Instruct + +Deploy [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-Next-80B-A3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.5.3rc1-cu126` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **128** +- Streaming: **enabled** diff --git a/qwen/qwen-3-next-80B-A3-instruct-sglang/config.yaml b/llm/qwen/qwen-3-next-80B-A3-instruct-sglang/config.yaml similarity index 96% rename from qwen/qwen-3-next-80B-A3-instruct-sglang/config.yaml rename to llm/qwen/qwen-3-next-80B-A3-instruct-sglang/config.yaml index 5c67bb964..4e1a6bcc7 100644 --- a/qwen/qwen-3-next-80B-A3-instruct-sglang/config.yaml +++ b/llm/qwen/qwen-3-next-80B-A3-instruct-sglang/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-Next-80B-A3B-Instruct for text generation" base_image: #image: lmsysorg/sglang@sha256:c977d3c5cf66029c8c37436a777381cb5bc861527da1b405c90ae2360417eedb image: lmsysorg/sglang:v0.5.3rc1-cu126 diff --git a/llm/qwen/qwen-3-next-80B-A3-thinking-sglang/README.md b/llm/qwen/qwen-3-next-80B-A3-thinking-sglang/README.md new file mode 100644 index 000000000..4ecc99334 --- /dev/null +++ b/llm/qwen/qwen-3-next-80B-A3-thinking-sglang/README.md @@ -0,0 +1,56 @@ +# Qwen3-Next-80B-A3B-Thinking + +Deploy [Qwen/Qwen3-Next-80B-A3B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-Next-80B-A3B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-Next-80B-A3B-Thinking-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.5.3rc1-cu126` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **128** +- Streaming: **enabled** diff --git a/qwen/qwen-3-next-80B-A3-thinking-sglang/config.yaml b/llm/qwen/qwen-3-next-80B-A3-thinking-sglang/config.yaml similarity index 95% rename from qwen/qwen-3-next-80B-A3-thinking-sglang/config.yaml rename to llm/qwen/qwen-3-next-80B-A3-thinking-sglang/config.yaml index e27892840..83015b226 100644 --- a/qwen/qwen-3-next-80B-A3-thinking-sglang/config.yaml +++ b/llm/qwen/qwen-3-next-80B-A3-thinking-sglang/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 for text generation" base_image: #image: lmsysorg/sglang@sha256:c977d3c5cf66029c8c37436a777381cb5bc861527da1b405c90ae2360417eedb image: lmsysorg/sglang:v0.5.3rc1-cu126 diff --git a/llm/qwen/qwen-3-vl-30b-a3b-instruct/README.md b/llm/qwen/qwen-3-vl-30b-a3b-instruct/README.md new file mode 100644 index 000000000..47ec50bda --- /dev/null +++ b/llm/qwen/qwen-3-vl-30b-a3b-instruct/README.md @@ -0,0 +1,54 @@ +# Qwen3-VL-30B-A3B-Instruct-FP8 + +Deploy [Qwen/Qwen3-VL-30B-A3B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct-FP8) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-VL-30B-A3B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct-FP8) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:2f7dbc9b42c51ba192e3dded515e4e07cdfdabea` +- Predict concurrency: **32** diff --git a/qwen/qwen-3-vl-30b-a3b-instruct/config.yaml b/llm/qwen/qwen-3-vl-30b-a3b-instruct/config.yaml similarity index 92% rename from qwen/qwen-3-vl-30b-a3b-instruct/config.yaml rename to llm/qwen/qwen-3-vl-30b-a3b-instruct/config.yaml index 87335d34c..dcc2adf3b 100644 --- a/qwen/qwen-3-vl-30b-a3b-instruct/config.yaml +++ b/llm/qwen/qwen-3-vl-30b-a3b-instruct/config.yaml @@ -1,4 +1,6 @@ +description: "Qwen3 VL 30B A3B Instruct for vision-language tasks" model_metadata: + repo_id: "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8" example_model_input: # Loads sample request into Baseten playground model: "Qwen/Qwen3-VL-30B-A3B-Thinking" stream: false diff --git a/llm/qwen/qwen-3-vl-30b-a3b-thinking/README.md b/llm/qwen/qwen-3-vl-30b-a3b-thinking/README.md new file mode 100644 index 000000000..b81c4b27c --- /dev/null +++ b/llm/qwen/qwen-3-vl-30b-a3b-thinking/README.md @@ -0,0 +1,54 @@ +# Qwen3-VL-30B-A3B-Thinking-FP8 + +Deploy [Qwen/Qwen3-VL-30B-A3B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Thinking-FP8) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-VL-30B-A3B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Thinking-FP8) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-VL-30B-A3B-Thinking-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-VL-30B-A3B-Thinking-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:2f7dbc9b42c51ba192e3dded515e4e07cdfdabea` +- Predict concurrency: **32** diff --git a/qwen/qwen-3-vl-30b-a3b-thinking/config.yaml b/llm/qwen/qwen-3-vl-30b-a3b-thinking/config.yaml similarity index 92% rename from qwen/qwen-3-vl-30b-a3b-thinking/config.yaml rename to llm/qwen/qwen-3-vl-30b-a3b-thinking/config.yaml index 8d2bf99fe..347f77999 100644 --- a/qwen/qwen-3-vl-30b-a3b-thinking/config.yaml +++ b/llm/qwen/qwen-3-vl-30b-a3b-thinking/config.yaml @@ -1,4 +1,6 @@ +description: "Qwen3 VL 30B A3B Thinking for vision-language tasks" model_metadata: + repo_id: "Qwen/Qwen3-VL-30B-A3B-Thinking-FP8" example_model_input: # Loads sample request into Baseten playground model: "Qwen/Qwen3-VL-30B-A3B-Thinking" stream: false diff --git a/llm/qwen/qwen-3-vl-32b/README.md b/llm/qwen/qwen-3-vl-32b/README.md new file mode 100644 index 000000000..07125e980 --- /dev/null +++ b/llm/qwen/qwen-3-vl-32b/README.md @@ -0,0 +1,57 @@ +# Qwen 3 VL 32B + +Deploy [Qwen/Qwen3-VL-32B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct-FP8) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-VL-32B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct-FP8) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:1 | +| OpenAI compatible | Yes | +| Python | py312 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-VL-32B-Instruct-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-VL-32B-Instruct-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `vllm/vllm-openai:v0.11.0` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **16** +- Streaming: **enabled** diff --git a/qwen/qwen-3-vl-32b/config.yaml b/llm/qwen/qwen-3-vl-32b/config.yaml similarity index 87% rename from qwen/qwen-3-vl-32b/config.yaml rename to llm/qwen/qwen-3-vl-32b/config.yaml index cd7cfcd2a..2d0b6553c 100644 --- a/qwen/qwen-3-vl-32b/config.yaml +++ b/llm/qwen/qwen-3-vl-32b/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen3 VL 32B Instruct for vision-language tasks" base_image: image: vllm/vllm-openai:v0.11.0 model_metadata: @@ -17,11 +18,11 @@ model_metadata: model_name: Qwen 3 VL 32B requirements: - transformers>=4.55.0 - - accelerate - - timm - - einops - - open-clip-torch - - pillow + - accelerate==1.2.1 + - timm==1.0.12 + - einops==0.8.0 + - open-clip-torch==2.29.0 + - pillow==10.4.0 python_version: py312 model_cache: - repo_id: Qwen/Qwen3-VL-32B-Instruct-FP8 diff --git a/llm/qwen/qwen-coder-next/README.md b/llm/qwen/qwen-coder-next/README.md new file mode 100644 index 000000000..10535d7ed --- /dev/null +++ b/llm/qwen/qwen-coder-next/README.md @@ -0,0 +1,56 @@ +# Qwen3-Coder-Next + +Deploy [Qwen/Qwen3-Coder-Next-FP8](https://huggingface.co/Qwen/Qwen3-Coder-Next-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-Coder-Next-FP8](https://huggingface.co/Qwen/Qwen3-Coder-Next-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-Coder-Next-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-Coder-Next-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:nightly-dev-20260202-9227d4f7` +- Predict concurrency: **64** +- System packages: `tmux, htop, nload` +- Streaming: **enabled** diff --git a/qwen/qwen-coder-next/config.yaml b/llm/qwen/qwen-coder-next/config.yaml similarity index 92% rename from qwen/qwen-coder-next/config.yaml rename to llm/qwen/qwen-coder-next/config.yaml index 92126518b..56482c0ab 100644 --- a/qwen/qwen-coder-next/config.yaml +++ b/llm/qwen/qwen-coder-next/config.yaml @@ -1,4 +1,6 @@ +description: "Qwen3 Coder Next for code generation" model_metadata: + repo_id: "Qwen/Qwen3-Coder-Next-FP8" example_model_input: messages: - role: system diff --git a/llm/qwen/qwen-image/README.md b/llm/qwen/qwen-image/README.md new file mode 100644 index 000000000..7ad9803ed --- /dev/null +++ b/llm/qwen/qwen-image/README.md @@ -0,0 +1,37 @@ +# Qwen Image + +Deploy [Qwen/Qwen-Image](https://huggingface.co/Qwen/Qwen-Image) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen-Image](https://huggingface.co/Qwen/Qwen-Image) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | H100 | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A beautiful sunset over a mountain landscape with golden clouds, Ultra HD, 4K, cinematic composition", + "width": 1024, + "height": 1024, + "num_inference_steps": 50, + "true_cfg_scale": 4.0, + "seed": 42 +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/qwen/qwen-image/assets/generated_image.jpg b/llm/qwen/qwen-image/assets/generated_image.jpg similarity index 100% rename from qwen/qwen-image/assets/generated_image.jpg rename to llm/qwen/qwen-image/assets/generated_image.jpg diff --git a/qwen/qwen-image/config.yaml b/llm/qwen/qwen-image/config.yaml similarity index 84% rename from qwen/qwen-image/config.yaml rename to llm/qwen/qwen-image/config.yaml index 7c1f200ff..514c3bb13 100644 --- a/qwen/qwen-image/config.yaml +++ b/llm/qwen/qwen-image/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen Image for vision-language tasks" external_package_dirs: [] model_cache: - repo_id: Qwen/Qwen-Image @@ -23,10 +24,10 @@ requirements: - git+https://github.com/huggingface/diffusers - torch>=2.5.1 - transformers>=4.48.2 - - accelerate - - safetensors - - pillow - - numpy + - accelerate==1.2.1 + - safetensors==0.4.1 + - pillow==10.4.0 + - numpy==1.26.0 resources: accelerator: H100 use_gpu: true diff --git a/qwen/qwen-image/model/model.py b/llm/qwen/qwen-image/model/model.py similarity index 100% rename from qwen/qwen-image/model/model.py rename to llm/qwen/qwen-image/model/model.py diff --git a/llm/qwen/qwen-vl/README.md b/llm/qwen/qwen-vl/README.md new file mode 100644 index 000000000..d178adc65 --- /dev/null +++ b/llm/qwen/qwen-vl/README.md @@ -0,0 +1,30 @@ +# Qwen VL + +Deploy [Qwen/Qwen-VL](https://huggingface.co/Qwen/Qwen-VL) for text generation using a Custom (Truss) engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen-VL](https://huggingface.co/Qwen/Qwen-VL) | +| Task | Text generation | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "What is machine learning?", "max_tokens": 512}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/qwen/qwen-vl/config.yaml b/llm/qwen/qwen-vl/config.yaml similarity index 69% rename from qwen/qwen-vl/config.yaml rename to llm/qwen/qwen-vl/config.yaml index 879020997..622219b40 100644 --- a/qwen/qwen-vl/config.yaml +++ b/llm/qwen/qwen-vl/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen VL for vision-language tasks" environment_variables: {} external_package_dirs: [] model_cache: @@ -9,6 +10,8 @@ model_cache: - '*.py' repo_id: Qwen/Qwen-VL use_volume: false +model_metadata: + example_model_input: {"image": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", "prompt": "Describe this image in detail"} model_name: Qwen VL python_version: py310 requirements: diff --git a/whisper/faster-whisper-v2/model/__init__.py b/llm/qwen/qwen-vl/model/__init__.py similarity index 100% rename from whisper/faster-whisper-v2/model/__init__.py rename to llm/qwen/qwen-vl/model/__init__.py diff --git a/qwen/qwen-vl/model/model.py b/llm/qwen/qwen-vl/model/model.py similarity index 100% rename from qwen/qwen-vl/model/model.py rename to llm/qwen/qwen-vl/model/model.py diff --git a/llm/seed/seed-llm/README.md b/llm/seed/seed-llm/README.md new file mode 100644 index 000000000..0d2cd8ffb --- /dev/null +++ b/llm/seed/seed-llm/README.md @@ -0,0 +1,56 @@ +# Seed-OSS-36B-Instruct + +Deploy [ByteDance-Seed/Seed-OSS-36B-Instruct](https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct) for text generation using a vLLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [ByteDance-Seed/Seed-OSS-36B-Instruct](https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct) | +| Task | Text generation | +| Engine | vLLM | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="ByteDance-Seed/Seed-OSS-36B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "ByteDance-Seed/Seed-OSS-36B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:78336a0c3ee4eb9dba6e37959d926160e91623fd` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **128** +- Streaming: **enabled** diff --git a/seed/seed_llm/config.yaml b/llm/seed/seed-llm/config.yaml similarity index 95% rename from seed/seed_llm/config.yaml rename to llm/seed/seed-llm/config.yaml index ef7027cb4..445c2dd58 100644 --- a/seed/seed_llm/config.yaml +++ b/llm/seed/seed-llm/config.yaml @@ -1,3 +1,4 @@ +description: "ByteDance-Seed/Seed-OSS-36B-Instruct for text generation" base_image: image: public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:78336a0c3ee4eb9dba6e37959d926160e91623fd build_commands: diff --git a/llm/z-ai/glm-4-5-air-fp8/README.md b/llm/z-ai/glm-4-5-air-fp8/README.md new file mode 100644 index 000000000..30ed43ef8 --- /dev/null +++ b/llm/z-ai/glm-4-5-air-fp8/README.md @@ -0,0 +1,56 @@ +# GLM 4.5 Air FP8 + +Deploy [zai-org/GLM-4.5-Air-FP8](https://huggingface.co/zai-org/GLM-4.5-Air-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [zai-org/GLM-4.5-Air-FP8](https://huggingface.co/zai-org/GLM-4.5-Air-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:4 | +| Quantization | FP8 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="zai-org/GLM-4.5-Air-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "zai-org/GLM-4.5-Air-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.9.post6-cu126` +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/z-ai/glm-4-5-air-fp8/config.yaml b/llm/z-ai/glm-4-5-air-fp8/config.yaml similarity index 91% rename from z-ai/glm-4-5-air-fp8/config.yaml rename to llm/z-ai/glm-4-5-air-fp8/config.yaml index c1408600b..d669acd48 100644 --- a/z-ai/glm-4-5-air-fp8/config.yaml +++ b/llm/z-ai/glm-4-5-air-fp8/config.yaml @@ -1,4 +1,6 @@ +description: "zai-org/GLM-4.5-Air-FP8 for text generation" model_metadata: + repo_id: "zai-org/GLM-4.5-Air-FP8" example_model_input: # Loads sample request into Baseten playground messages: - role: system diff --git a/llm/z-ai/glm-4-5-fp8/README.md b/llm/z-ai/glm-4-5-fp8/README.md new file mode 100644 index 000000000..cfe5e3308 --- /dev/null +++ b/llm/z-ai/glm-4-5-fp8/README.md @@ -0,0 +1,56 @@ +# GLM 4.5 FP8 + +Deploy [zai-org/GLM-4.5-FP8](https://huggingface.co/zai-org/GLM-4.5-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [zai-org/GLM-4.5-FP8](https://huggingface.co/zai-org/GLM-4.5-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:8 | +| Quantization | FP8 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="zai-org/GLM-4.5-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "zai-org/GLM-4.5-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.4.9.post6-cu126` +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/z-ai/glm-4-5-fp8/config.yaml b/llm/z-ai/glm-4-5-fp8/config.yaml similarity index 92% rename from z-ai/glm-4-5-fp8/config.yaml rename to llm/z-ai/glm-4-5-fp8/config.yaml index f2c1177dc..df10ead3c 100644 --- a/z-ai/glm-4-5-fp8/config.yaml +++ b/llm/z-ai/glm-4-5-fp8/config.yaml @@ -1,4 +1,6 @@ +description: "zai-org/GLM-4.5-FP8 for text generation" model_metadata: + repo_id: "zai-org/GLM-4.5-FP8" example_model_input: # Loads sample request into Baseten playground messages: - role: system diff --git a/llm/z-ai/glm-4-5-v/README.md b/llm/z-ai/glm-4-5-v/README.md new file mode 100644 index 000000000..5e66e3a3a --- /dev/null +++ b/llm/z-ai/glm-4-5-v/README.md @@ -0,0 +1,54 @@ +# GLM-4.5V-FP8 + +Deploy [zai-org/GLM-4.5V-FP8](https://huggingface.co/zai-org/GLM-4.5V-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [zai-org/GLM-4.5V-FP8](https://huggingface.co/zai-org/GLM-4.5V-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:8 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="zai-org/GLM-4.5V-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "zai-org/GLM-4.5V-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.5.4.post1-cu129-amd64` +- Predict concurrency: **32** diff --git a/z-ai/glm_4_5_v/config.yaml b/llm/z-ai/glm-4-5-v/config.yaml similarity index 92% rename from z-ai/glm_4_5_v/config.yaml rename to llm/z-ai/glm-4-5-v/config.yaml index de546b83c..3a10ddd00 100644 --- a/z-ai/glm_4_5_v/config.yaml +++ b/llm/z-ai/glm-4-5-v/config.yaml @@ -1,4 +1,6 @@ +description: "GLM-4.5V for vision-language tasks" model_metadata: + repo_id: "zai-org/GLM-4.5V-FP8" example_model_input: # Loads sample request into Baseten playground messages: - role: system diff --git a/llm/z-ai/glm-4-6-fp8/README.md b/llm/z-ai/glm-4-6-fp8/README.md new file mode 100644 index 000000000..5ddcaa9b8 --- /dev/null +++ b/llm/z-ai/glm-4-6-fp8/README.md @@ -0,0 +1,56 @@ +# GLM-4.6-FP8 + +Deploy [zai-org/GLM-4.6-FP8](https://huggingface.co/zai-org/GLM-4.6-FP8) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [zai-org/GLM-4.6-FP8](https://huggingface.co/zai-org/GLM-4.6-FP8) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:8 | +| Quantization | FP8 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="zai-org/GLM-4.6-FP8", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "zai-org/GLM-4.6-FP8", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:v0.5.3rc1-cu126` +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/z-ai/glm-4-6-fp8/config.yaml b/llm/z-ai/glm-4-6-fp8/config.yaml similarity index 93% rename from z-ai/glm-4-6-fp8/config.yaml rename to llm/z-ai/glm-4-6-fp8/config.yaml index 140d2e125..89214233d 100644 --- a/z-ai/glm-4-6-fp8/config.yaml +++ b/llm/z-ai/glm-4-6-fp8/config.yaml @@ -1,4 +1,6 @@ +description: "zai-org/GLM-4.6-FP8 for text generation" model_metadata: + repo_id: "zai-org/GLM-4.6-FP8" example_model_input: # Loads sample request into Baseten playground messages: - role: system diff --git a/llm/z-ai/glm-4-7-flash/README.md b/llm/z-ai/glm-4-7-flash/README.md new file mode 100644 index 000000000..88978ae85 --- /dev/null +++ b/llm/z-ai/glm-4-7-flash/README.md @@ -0,0 +1,55 @@ +# GLM 4.7 Flash + +Deploy [zai-org/GLM-4.7-Flash](https://huggingface.co/zai-org/GLM-4.7-Flash) for text generation using a SGLang engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [zai-org/GLM-4.7-Flash](https://huggingface.co/zai-org/GLM-4.7-Flash) | +| Task | Text generation | +| Engine | SGLang | +| GPU | H100:2 | +| OpenAI compatible | Yes | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="zai-org/GLM-4.7-Flash", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "zai-org/GLM-4.7-Flash", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Base image: `lmsysorg/sglang:nightly-dev-20260122-e6ccb294` +- Predict concurrency: **32** +- Streaming: **enabled** diff --git a/z-ai/glm_4_7_flash/config.yaml b/llm/z-ai/glm-4-7-flash/config.yaml similarity index 92% rename from z-ai/glm_4_7_flash/config.yaml rename to llm/z-ai/glm-4-7-flash/config.yaml index 0997ba76b..e58b9f2ca 100644 --- a/z-ai/glm_4_7_flash/config.yaml +++ b/llm/z-ai/glm-4-7-flash/config.yaml @@ -1,4 +1,6 @@ +description: "zai-org/GLM-4.7-Flash for text generation" model_metadata: + repo_id: "zai-org/GLM-4.7-Flash" example_model_input: messages: - role: system diff --git a/lora/engine-lora/README.md b/lora/engine-lora/README.md deleted file mode 100644 index 9f22e950b..000000000 --- a/lora/engine-lora/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# IMPORTANT NOTICE - -Currently we're seeing an issue where using LoRAs with TensorRT-LLM shows a significant drop in performance. It is strongly recommended to go with the [vLLM example](../vllm-lora) - -# Mistral 7B Instruct LoRA - -This is an example of a truss that supports **dynamic swapping of LoRA adapters**β€”allowing you to serve multiple fine-tuned variants efficiently from a single GPU. In this example, we deploy a **Mistral 7B Instruct** on [TensorRT-LLM Engine Builder](https://docs.baseten.co/performance/examples/mistral-trt). This model will be an expert in finance, medicine and law. - -- πŸ“„ **TensorRT-LLM Details:** [Performance Example (Baseten Docs)](https://docs.baseten.co/performance/examples/mistral-trt) -- πŸ’‘ **LoRA Swapping Overview:** [Baseten Blog: Serving 10,000 Fine-Tuned LLMs from One GPU](https://www.baseten.co/blog/how-to-serve-10-000-fine-tuned-llms-from-a-single-gpu/) - ---- - -## πŸ› οΈ Implementing LoRA Swapping - -Extending a base TensorRT-LLM deployment to support LoRA swapping requires three config changes: - -### 1. Configure `lora_adapters` - -List each LoRA adapter’s name and its download source. Supported sources are: -- `HF` for HuggingFace -- `GCS` for Google Cloud Storage -- `REMOTE_URL` for any direct download - -**Example (`config.yaml`):** -```yaml -lora_adapters: - legal: - source: HF - repo: Aretoss/Lexgen - finance: - source: HF - repo: vaibhav1/lora-mistral-finance - medical: - source: HF - repo: Imsachinsingh00/Fine_tuned_LoRA_Mistral_MTSDialog_Summarization -``` - -### 2. Set `served_model_name` (Optional) - -Set this parameter if you wish to allow requests to the base model (without any LoRA applied). - ---- - -### 3. Select Adapter or Base Model at Request Time - -Specify the desired adapter or base model using the `model` field in your request payload. - -**Example request body** - -```json -{ - "model": "finance", // # Or legal, medical, or mistral - "stream": true, - "messages": [ - {"role": "user", "content": "What would you choose in 2008?"} - ], - "max_tokens": 1024, - "temperature": 0.9 -} -``` - -### For full details, see [documentation](https://docs.baseten.co/development/model/performance/engine-builder-config) diff --git a/lora/sglang-lora/README.md b/lora/sglang-lora/README.md deleted file mode 100644 index 58a383f40..000000000 --- a/lora/sglang-lora/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# IMPORTANT NOTICE - -Currently, [support for openai compatible server with lora is not yet implemented in SGLang](https://github.com/sgl-project/sglang/issues/2929). It is strongly recommended to go with the [vLLM example](../vllm-lora). - -# Mistral 7B Instruct LoRA - -This is an example of a truss that supports **dynamic swapping of LoRA adapters**β€”allowing you to serve multiple fine-tuned variants efficiently from a single GPU. In this example, we deploy a **Mistral 7B Instruct** with SGLang. This model will be an expert in finance, medicine and law. - -- πŸ’‘ **LoRA Swapping Overview:** [Baseten Blog: Serving 10,000 Fine-Tuned LLMs from One GPU](https://www.baseten.co/blog/how-to-serve-10-000-fine-tuned-llms-from-a-single-gpu/) - ---- - -## πŸ› οΈ Implementing LoRA Swapping - -Extending a base SGLang custom server deployment to support LoRA swapping requires two config changes: - -### 1. Configure `lora_paths` - -List each LoRA adapter’s name and its huggingface repo. - -**Example (`start_command` in `config.yaml`):** -``` ---enable-lora --lora-paths legal=Aretoss/Lexgen finance=vaibhav1/lora-mistral-finance medical=Imsachinsingh00/Fine_tuned_LoRA_Mistral_MTSDialog_Summarization --disable-radix-cache -``` - -Note that the `--disable-radix-cache` flag is necessary because lora with radix attention is not yet implemented in SGLang. - ---- - -### 3. Select Adapter or Base Model at Request Time - -This API is not openai compatible. Follow the example model input in the `README.md`. - -**Example request body** - -```json -{ -"text": [ - "What would you choose in 2008?", - "What would you choose in 2008?", -], -"sampling_params": {"max_new_tokens": 1000, "temperature": 1.0}, -"lora_path": ["legal", "finance"], -} -``` - -### For full details, see SGLang's [documentation](https://docs.sglang.ai/backend/lora.html) diff --git a/lora/vllm-lora/README.md b/lora/vllm-lora/README.md deleted file mode 100644 index 15b75b7f0..000000000 --- a/lora/vllm-lora/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Mistral 7B Instruct LoRA - -This is an example of a truss that supports **dynamic swapping of LoRA adapters**β€”allowing you to serve multiple fine-tuned variants efficiently from a single GPU. In this example, we deploy a **Mistral 7B Instruct** on vLLM's server. This model will be an expert in finance and law. - -- πŸ’‘ **LoRA Swapping Overview:** [Baseten Blog: Serving 10,000 Fine-Tuned LLMs from One GPU](https://www.baseten.co/blog/how-to-serve-10-000-fine-tuned-llms-from-a-single-gpu/) - ---- - -## πŸ› οΈ Implementing LoRA Swapping - -Extending a base vLLM deployment to support LoRA swapping requires three config changes: - -### 1. Configure `lora_modules` - -List each LoRA adapter’s name and its huggingface repo. - -**Example (`config.yaml`):** -``` ---enable-lora --lora-modules finance=vaibhav1/lora-mistral-finance legal=Aretoss/Lexgen -``` - -### 2. Set `served_model_name` (Optional) - -Set this parameter if you wish to allow requests to the base model (without any LoRA applied). - ---- - -### 3. Select Adapter or Base Model at Request Time - -Specify the desired adapter or base model using the `model` field in your request payload. - -**Example request body** - -```json -{ - "model": "finance", // Or "legal" or "mistral" - "stream": true, - "messages": [ - {"role": "user", "content": "What would you choose in 2008?"} - ], - "max_tokens": 1024, - "temperature": 0.9 -} -``` - -### For full details, see [documentation](https://docs.vllm.ai/en/v0.9.1/features/lora.html#lora-model-lineage-in-model-card) diff --git a/magic-animate/README.md b/magic-animate/README.md deleted file mode 100644 index a07290037..000000000 --- a/magic-animate/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Magic Animate Truss - -This repository packages [Magic Animate](https://github.com/magic-research/magic-animate) as a [Truss](https://truss.baseten.co/). - -Magic Animate allows you to create human image animation using a diffusion model. The model combines two inputs, a picture of a person and a densepose motion sequence, so that the human in the picture gets animated based on the provided motion sequence. Here is an example: - - -https://github.com/htrivedi99/truss-examples/assets/15642666/914d7b50-c0e3-40fc-a146-c53743c82cbd - - -## Deploying Magic Animate - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd magic-animate -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `magic-animate` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking the model - -Here are the following inputs for the model: -1. `reference_image` (required): The image of the person you'd like to animate as a base64 string. Square images work better since each image gets resized to 512 x 512. -2. `motion_sequence` (required): A densepose motion sequence as a base64 string. You can use something like [detectron2](https://github.com/facebookresearch/detectron2/tree/main/projects/DensePose) to create custom densepose sequences. -3. `seed` (optional): A random seed for the model. -4. `steps` (optional): The number of iterations the model runs through. -5. `guidance_scale` (optional): Used to determine how closely the image generation follows the prompt. -6. `grid` (optional): A boolean value which controls if the output comes back with all the clips or just the individual output. - -If `grid` is passed in as `True` the model will respond with a JSON object with two keys: `output` and `grid_clip`. If `grid` is not passed in or is set to `False` the JSON object returned only has one key `output`. Both `output` and `grid_clip` are MP4 files returned as a base64 string. - -Here is an example of how to invoke this model: - -```python -from PIL import Image -import base64 - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -def mp4_to_base64(file_path: str): - with open(file_path, "rb") as mp4_file: - binary_data = mp4_file.read() - base64_data = base64.b64encode(binary_data) - base64_string = base64_data.decode("utf-8") - - return base64_string - -def base64_to_mp4(base64_string, output_file_path): - binary_data = base64.b64decode(base64_string) - with open(output_file_path, "wb") as output_file: - output_file.write(binary_data) - -img = Image.open("/path/to/image/monalisa.png") -input_img = pil_to_b64(img) -motion_sequence = mp4_to_base64("/path/to/densepose/sequence/demo4.mp4") -data = {"reference_image": input_img, "motion_sequence": motion_sequence, "steps": 10, "grid": True} -res = requests.post("https://model-.api.baseten.co/development/predict", headers=headers, json=data) -res = res.json() -base64_to_mp4(res.get("output"), "magic-animate.mp4") -base64_to_mp4(res.get("grid_clip"), "grid.mp4") -``` - -Here is the example `monalisa.png` image: - -![monalisa](https://github.com/htrivedi99/truss-examples/assets/15642666/9e9f4e40-6c55-415b-b37c-3271572ffb77) - - -Here is the example densepose sequence `demo4.mp4`: - -https://github.com/htrivedi99/truss-examples/assets/15642666/c20a9761-1279-4c0b-9de1-7fd73cb43fd7 diff --git a/metavoice-1b/README.md b/metavoice-1b/README.md deleted file mode 100644 index b58075bbe..000000000 --- a/metavoice-1b/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# MetaVoice 1B Truss -[MetaVoice-1B](https://github.com/metavoiceio/metavoice-src) is an Apache-licensed, 1.2B parameter base model trained on 100K hours of speech for TTS (text-to-speech). - -This model is packaged using [Truss](https://trussml.com), the simplest way to serve AI/ML models in production. - -## Deploy MetaVoice 1B -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd metavoice-1b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `metavoice-1b` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using MetaVoice through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - - -## Invoking MetaVoice - -To use MetaVoice 1B, follow this command pattern, keeping in mind the 220-character limit for input text: - -```sh -truss predict -d '{"text": "Your input text here"}' | python process.py -``` - -### Understanding process.py -The process.py script is essential for handling the output from MetaVoice 1B. It reads the base64 encoded audio from the standard input, decodes it, and saves it as a WAV file. Here's a brief overview of how it works: - -```python -import base64 -import sys - -b64_audio = sys.stdin.read() -b64_audio = b64_audio.split('"')[1] # Extracting the base64 string - -wav_file = open("output.wav", "wb") -decode_string = base64.b64decode(b64_audio) -wav_file.write(decode_string) -``` - -## Notes -- `flash_attn` requires installation with `--no-build-isolation`. As this isn't supported, installing the wheel directly seems to work (see `config.yaml`). - -## To Dos -- Add support for passing additional reference voices to use diff --git a/midnight/README.md b/midnight/README.md deleted file mode 100644 index d25492e59..000000000 --- a/midnight/README.md +++ /dev/null @@ -1,254 +0,0 @@ -# Kaiko Midnight - Pathology Foundation Model - -A Baseten deployment of the [Kaiko Midnight](https://huggingface.co/kaiko-ai/midnight) pathology foundation model for medical image analysis and classification. - -## Overview - -Kaiko Midnight is a 1.14B parameter pathology foundation model based on DINOv2, optimized for medical image analysis. It provides both classification and segmentation embeddings for pathology images. - -## Deploy Kiako Midnight -First, clone this repository: - -```bash -git clone https://github.com/basetenlabs/truss-examples/ -cd midnight -``` - -Before deployment: - -Make sure you have a Baseten account and API key. -Install the latest version of Truss: `pip install --upgrade truss` -With midnight as your working directory, you can deploy the model with: - -```bash -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see Truss documentation. - -Once your Truss is deployed, you can start using Midnight through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -Note: If you run into the following error during the build phase, downgrade truss with: `pip install truss==0.9.111` - -``` -error: failed to solve: process "/bin/sh -c uv pip install --python $(which python3) -r base_server_requirements.txt --no-cache-dir" did not complete successfully: exit code: 2 -``` - -## GPU Requirements - -- **Minimum**: T4 (16GB VRAM) - -## API Usage - -#### Single Image Processing -```json -{ - "image_url": "https://upload.wikimedia.org/wikipedia/commons/8/80/Breast_DCIS_histopathology_%281%29.jpg", - "task": "classification", - "batch_size": 1 -} -``` - -#### Single Image with Base64 -```json -{ - "image_base64": "iVBORw0KGgoAAAANSUhEUgAA...", - "task": "classification", - "batch_size": 1 -} -``` - -#### Batch Processing (Multiple Images) -```json -{ - "image_urls": [ - "https://example.com/image1.jpg", - "https://example.com/image2.jpg", - "https://example.com/image3.jpg", - "https://example.com/image4.jpg" - ], - "task": "classification", - "batch_size": 4 -} -``` - -#### True Batch Processing with Base64 -```json -{ - "image_base64_list": [ - "iVBORw0KGgoAAAANSUhEUgAA...", - "iVBORw0KGgoAAAANSUhEUgAA...", - "iVBORw0KGgoAAAANSUhEUgAA...", - "iVBORw0KGgoAAAANSUhEUgAA..." - ], - "task": "classification", - "batch_size": 4 -} -``` - -### Response Format -```json -{ - "embeddings": [[0.123, 0.456, ...], [0.789, 0.012, ...]], - "embedding_shape": [2, 3072], - "task": "classification", - "model_id": "kaiko-ai/midnight", - "input_size": 224, - "actual_batch_size": 2, - "requested_batch_size": 4, - "optimal_batch_size": 8, - "gpu_memory_gb": 16.0 -} -``` - -## Example Usage - -### Python Client - Single Image -```python -import requests -import base64 - -# Baseten endpoint URL -endpoint_url = "https://your-baseten-endpoint.baseten.co" - -# Example request for classification -request_data = { - "image_url": "https://upload.wikimedia.org/wikipedia/commons/8/80/Breast_DCIS_histopathology_%281%29.jpg", - "task": "classification", - "batch_size": 1 -} - -# Make prediction -response = requests.post( - endpoint_url, - headers={"Authorization": "Api-Key YOUR_API_KEY"}, - json=request_data -) -result = response.json() - -print(f"Task: {result['task']}") -print(f"Embedding shape: {result['embedding_shape']}") -print(f"Embedding (first 10 values): {result['embeddings'][0][:10]}") -``` - -### Python Client - True Batch Processing -```python -import requests -import base64 - -def encode_image_to_base64(image_path): - """Convert local image to base64 string""" - with open(image_path, "rb") as image_file: - return base64.b64encode(image_file.read()).decode('utf-8') - -# Baseten endpoint URL -endpoint_url = "https://your-baseten-endpoint.baseten.co" - -# Encode multiple local images -image_paths = ["path/to/image1.jpg", "path/to/image2.jpg", "path/to/image3.jpg", "path/to/image4.jpg"] -image_base64_list = [encode_image_to_base64(path) for path in image_paths] - -# Example request for batch classification -request_data = { - "image_base64_list": image_base64_list, - "task": "classification", - "batch_size": 4 -} - -# Make prediction -response = requests.post( - endpoint_url, - headers={"Authorization": "Api-Key YOUR_API_KEY"}, - json=request_data -) -result = response.json() - -print(f"Task: {result['task']}") -print(f"Embedding shape: {result['embedding_shape']}") -print(f"Actual batch size: {result['actual_batch_size']}") -print(f"Number of embeddings: {len(result['embeddings'])}") -``` - -### cURL Example - Single Image -```bash -curl -X POST "https://your-baseten-endpoint.baseten.co" \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "image_url": "https://upload.wikimedia.org/wikipedia/commons/8/80/Breast_DCIS_histopathology_%281%29.jpg", - "task": "classification", - "batch_size": 1 - }' -``` - -### cURL Example - True Batch Processing -```bash -# First, encode your images to base64 -IMAGE1_BASE64=$(base64 -i path/to/image1.jpg) -IMAGE2_BASE64=$(base64 -i path/to/image2.jpg) -IMAGE3_BASE64=$(base64 -i path/to/image3.jpg) -IMAGE4_BASE64=$(base64 -i path/to/image4.jpg) - -# Then make the batch request -curl -X POST "https://your-baseten-endpoint.baseten.co" \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -H "Content-Type: application/json" \ - -d "{ - \"image_base64_list\": [\"$IMAGE1_BASE64\", \"$IMAGE2_BASE64\", \"$IMAGE3_BASE64\", \"$IMAGE4_BASE64\"], - \"task\": \"classification\", - \"batch_size\": 4 - }" -``` - -## Input Requirements - -### Image Requirements -- **Format**: JPEG, PNG, TIFF, or any format supported by PIL -- **Size**: Any size (automatically resized to 224x224) -- **Channels**: RGB (automatically converted if needed) -- **Max Size**: Subject to Baseten's request size limits (typically 10MB) - -### Base64 Requirements -- **Encoding**: Standard base64 encoding -- **Data URL support**: Both raw base64 and data URLs (data:image/...) are supported -- **Size limit**: Subject to Baseten's request size limits (typically 10MB) -- **Format**: Any image format supported by PIL - -### Batch Processing Requirements -- **True Batch**: Process multiple different images in a single request -- **Batch Size**: Automatically optimized based on GPU memory -- **Image Count**: Should match or exceed requested batch_size for optimal performance -- **Memory Efficiency**: Better GPU utilization than single image processing - -## Task Types - -### Classification -- **Purpose**: Global image embeddings for classification tasks -- **Output**: Concatenated CLS token + mean patch embeddings -- **Shape**: `(batch_size, 3072)` - 1536 + 1536 dimensions -- **Use Cases**: Image classification, similarity search, feature extraction - -### Segmentation -- **Purpose**: Spatial embeddings for segmentation tasks -- **Output**: Patch embeddings reshaped to spatial dimensions -- **Shape**: `(batch_size, 1536, 16, 16)` - 16x16 spatial grid -- **Use Cases**: Semantic segmentation, object detection, spatial analysis - - - -## Model Architecture - -### Base Model -- **Architecture**: DINOv2 Vision Transformer -- **Parameters**: 1.14B parameters -- **Precision**: F32 (float32) -- **Input Size**: 224x224 pixels -- **Patch Size**: 14x14 pixels -- **Hidden Size**: 1536 dimensions - -### Embedding Extraction -- **Classification**: CLS token + mean patch embeddings (3072 dimensions) -- **Segmentation**: Patch embeddings reshaped to spatial grid (1536x16x16) -- **Normalization**: Mean=(0.5,0.5,0.5), Std=(0.5,0.5,0.5) diff --git a/mistral/engine-mistral-small-3/README.md b/mistral/engine-mistral-small-3/README.md deleted file mode 100644 index 4cbb6ba55..000000000 --- a/mistral/engine-mistral-small-3/README.md +++ /dev/null @@ -1 +0,0 @@ -# Mistral Small 3 (2501) diff --git a/mistral/engine-mixtral-8x22b-instruct/README.md b/mistral/engine-mixtral-8x22b-instruct/README.md deleted file mode 100644 index e41b6265c..000000000 --- a/mistral/engine-mixtral-8x22b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Mistral 8x22B Instruct - -This deployment of Mistral 8x22B Instruct uses the TensorRT-LLM Engine Builder. - -For details, see: https://docs.baseten.co/performance/examples/mistral-trt diff --git a/mistral/engine-mixtral-8x7b-instruct/README.md b/mistral/engine-mixtral-8x7b-instruct/README.md deleted file mode 100644 index 8f08fc2d6..000000000 --- a/mistral/engine-mixtral-8x7b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Mistral 8x7B Instruct - -This deployment of Mistral 8x7B Instruct uses the TensorRT-LLM Engine Builder. - -For details, see: https://docs.baseten.co/performance/examples/mistral-trt diff --git a/mistral/mistral-7b/README.md b/mistral/mistral-7b/README.md deleted file mode 100644 index 608985a26..000000000 --- a/mistral/mistral-7b/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Mistral 7B Truss - -This is a [Truss](https://truss.baseten.co/) for Mistral 7B. Mistral 7B parameter language model released by [Mistral](https://mistral.ai/) that outperforms other models in the 7B model class. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Mistral 7B. - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp)). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mistral-7b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mistral-7b` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -This seven billion parameter model is running in `float16` so that it fits on an A10G. - -## Mistral 7B API documentation - -This section provides an overview of the Mistral 7B API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __stream__: (optional, default=False) A boolean if the model should stream a response back. -- __max_new_tokens__ (optional, default=512): The maximum number of tokens to return, counting input tokens. Maximum of 4096. -- __temperature__ (optional, default=0.1): Controls the randomness of the generated text. Higher values produce more diverse results, while lower values produce more deterministic results. -- __top_p__ (optional, default=0.75): The cumulative probability threshold for token sampling. The model will only consider tokens whose cumulative probability is below this threshold. -- __top_k__ (optional, default=40): The number of top tokens to consider when sampling. The model will only consider the top_k highest-probability tokens. - -The API also supports passing any parameter supported by HuggingFace's `Transformers.generate`. - -## Example usage - -```sh -truss predict -d '{"prompt": "What is the meaning of life?", "max_new_tokens": 4096}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "What's the meaning of life?", - "max_new_tokens": 4096 - }' -``` diff --git a/mistral/mixtral-8x22b-trt-int8-weights-only/README.md b/mistral/mixtral-8x22b-trt-int8-weights-only/README.md deleted file mode 100644 index ee9591770..000000000 --- a/mistral/mixtral-8x22b-trt-int8-weights-only/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Mixtral 8x22B Instruct Truss - -This is a [Truss](https://truss.baseten.co/) for Mixtral 8x22B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Mistral 7B Instruct. - -**Warning: This example is only intended for usage on four A100 GPUs, changing your resource type for this deployment will result in unsupported behavior** - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten. Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -## Deploying Mixtral 8x22B Instruct - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mixtral-8x22b-trt-int8-weights-only -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x22b-trt-int8-weights-only` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Mixtral 8x22B Instruct API documentation -This section provides an overview of the Mistral 8x22B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided instruction. - -### API route: `predict` - -We expect requests will the following information: - -- ```messages``` (str): The prompt you'd like to complete -- ```max_tokens``` (int, default: 50): The max token count. This includes the number of tokens in your prompt so if this value is less than your prompt, you'll just recieve a truncated version of the prompt. -- ```temperature``` (float, default: 0.7): Determines the creativity of the model output. -- ```top_p``` (float, default: 0.7): Used to control model generation in conjunction with temperature. -- ```top_k``` (int, default: 50): Chooses the token based on a distribution of the top K tokens. -- ```beam_width``` (int, default:50): The number of beams to compute. This must be 1 for this version of TRT-LLM. Inflight-batching does not support beams > 1. -- ```bad_words_list``` (list, default:[]): A list of words to not include in generated output. -- ```stop_words_list``` (list, default:[]): A list of words to stop generation upon encountering. -- ```repetition_penalty``` (float, defualt: 1.0): A repetition penalty to incentivize not repeating tokens. - -This Truss will stream responses back. Responses will be buffered chunks of text. diff --git a/mistral/mixtral-8x22b/README.md b/mistral/mixtral-8x22b/README.md deleted file mode 100644 index 026856769..000000000 --- a/mistral/mixtral-8x22b/README.md +++ /dev/null @@ -1,78 +0,0 @@ -# Mixtral 8x22B Truss - -This is a [Truss](https://truss.baseten.co/) for the community edition of [Mixtral 8x22B](https://huggingface.co/mistral-community/Mixtral-8x22B-v0.1). -This is not an optimized model. If you would like to have a more optimized version that has lower latency + higher throughput, please contact our team. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mistral/mixtral-8x22b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x22b` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -You need four A100s to run Mixtral at `fp16`. If you need access to A100s, please [contact us](mailto:support@baseten.co). - -## Mixtral 8x22B API documentation - -This section provides an overview of the Mixtral 8x22B API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The `predict` route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __stream__ (optional, default=True): A boolean determining whether the model should stream a response back. When `True`, the API returns generated text as it becomes available. -- __max_tokens__ (optional, default=128): Determines the maximum number of tokens to generate -- __temperature__ (optional, default=1.0): Controls the strength of the generation. The higher the temperature, the more diverse and creative the output would be. -- __top_p__ (optional, default=0.95): Parameter used to control the randomness of the output. -- __top_k__ (optional, default=50): Controls the vocab size considered during the generation. - -## Example usage - -```python -import requests -import os - -# Replace the empty string with your model id below -model_id = "" -baseten_api_key = os.environ["BASETEN_API_KEY"] - -data = { - "prompt": "What is mistral wind?", - "stream": True, - "max_tokens": 256, - "temperature": 0.9 -} - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/production/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=data, - stream=True -) - -# Print the generated tokens as they get streamed -for content in res.iter_content(): - print(content.decode("utf-8"), end="", flush=True) -``` diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md b/mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md deleted file mode 100644 index 23e3f9142..000000000 --- a/mistral/mixtral-8x7b-instruct-trt-llm-h100/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Mixtral-8x7B-Instruct Truss - -This is a [Truss](https://truss.baseten.co/) for Mixtral 8x7B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Mistral 7B Instruct. - -**Warning: This example is intended for usage on two H100s, changing your resource type for this deployment will result in unsupported behavior** - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten. Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -## Deploying Mixtral-8x7B-Instruct - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mixtral-8x7b-instruct-trt-llm-h100 -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x7b-instruct-trt-llm-h100` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Mixtral 8x7B Instruct API documentation -This section provides an overview of the Mistral 7B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided instruction. - -### API route: `predict` - -This model is designed for our ChatCompletions endpoint: - -- [ChatCompletions endpoint tutorial](https://www.baseten.co/blog/gpt-vs-mistral-migrate-to-open-source-llms-with-minor-code-changes/) -- [ChatCompletions endpoint reference docs](https://docs.baseten.co/api-reference/openai) - -We expect requests will the following information: - -- ```messages``` (str): The prompt you'd like to complete -- ```max_tokens``` (int, default: 50): The max token count. This includes the number of tokens in your prompt so if this value is less than your prompt, you'll just recieve a truncated version of the prompt. -- ```beam_width``` (int, default:50): The number of beams to compute. This must be 1 for this version of TRT-LLM. Inflight-batching does not support beams > 1. -- ```bad_words_list``` (list, default:[]): A list of words to not include in generated output. -- ```stop_words_list``` (list, default:[]): A list of words to stop generation upon encountering. -- ```repetition_penalty``` (float, defualt: 1.0): A repetition penalty to incentivize not repeating tokens. - -This Truss will stream responses back. Responses will be buffered chunks of text. diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md b/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md deleted file mode 100644 index 3e4984352..000000000 --- a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Mixtral-8x7B-Instruct Truss - -This is a [Truss](https://truss.baseten.co/) for Mixtral 8x7B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Mistral 7B Instruct. - -**Warning: This example is only intended for usage on a single H100, changing your resource type for this deployment will result in unsupported behavior** - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten. Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -## Deploying Mixtral-8x7B-Instruct-Chat - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x7b-instruct-trt-llm-weights-only-quant` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Mixtral 8x7B Instruct API documentation -This section provides an overview of the Mistral 7B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided instruction. - -### API route: `predict` - -This model is designed for our ChatCompletions endpoint: - -- [ChatCompletions endpoint tutorial](https://www.baseten.co/blog/gpt-vs-mistral-migrate-to-open-source-llms-with-minor-code-changes/) -- [ChatCompletions endpoint reference docs](https://docs.baseten.co/api-reference/openai) - -We expect requests will the following information: - -- ```messages``` (str): The prompt you'd like to complete -- ```max_tokens``` (int, default: 50): The max token count. This includes the number of tokens in your prompt so if this value is less than your prompt, you'll just recieve a truncated version of the prompt. -- ```beam_width``` (int, default:50): The number of beams to compute. This must be 1 for this version of TRT-LLM. Inflight-batching does not support beams > 1. -- ```bad_words_list``` (list, default:[]): A list of words to not include in generated output. -- ```stop_words_list``` (list, default:[]): A list of words to stop generation upon encountering. -- ```repetition_penalty``` (float, defualt: 1.0): A repetition penalty to incentivize not repeating tokens. - -This Truss will stream responses back. Responses will be buffered chunks of text. diff --git a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md b/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md deleted file mode 100644 index 51acaa368..000000000 --- a/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Mixtral-8x7B-Instruct Truss - -This is a [Truss](https://truss.baseten.co/) for Mixtral 8x7B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Mistral 7B Instruct. - -**Warning: This example is only intended for usage on a single A100, changing your resource type for this deployment will result in unsupported behavior** - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten. Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -## Deploying Mixtral-8x7B-Instruct-Chat - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mixtral-8x7b-instruct-trt-llm-weights-only-quant -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x7b-instruct-trt-llm-weights-only-quant` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Mixtral 8x7B Instruct API documentation -This section provides an overview of the Mistral 7B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided instruction. - -### API route: `predict` - -This model is designed for our ChatCompletions endpoint: - -- [ChatCompletions endpoint tutorial](https://www.baseten.co/blog/gpt-vs-mistral-migrate-to-open-source-llms-with-minor-code-changes/) -- [ChatCompletions endpoint reference docs](https://docs.baseten.co/api-reference/openai) - -We expect requests will the following information: - -- ```messages``` (str): The prompt you'd like to complete -- ```max_tokens``` (int, default: 50): The max token count. This includes the number of tokens in your prompt so if this value is less than your prompt, you'll just recieve a truncated version of the prompt. -- ```beam_width``` (int, default:50): The number of beams to compute. This must be 1 for this version of TRT-LLM. Inflight-batching does not support beams > 1. -- ```bad_words_list``` (list, default:[]): A list of words to not include in generated output. -- ```stop_words_list``` (list, default:[]): A list of words to stop generation upon encountering. -- ```repetition_penalty``` (float, defualt: 1.0): A repetition penalty to incentivize not repeating tokens. - -This Truss will stream responses back. Responses will be buffered chunks of text. diff --git a/mistral/mixtral-8x7b-instruct-trt-llm/README.md b/mistral/mixtral-8x7b-instruct-trt-llm/README.md deleted file mode 100644 index e15f8d1ac..000000000 --- a/mistral/mixtral-8x7b-instruct-trt-llm/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Mixtral-8x7B-Instruct Truss - -This is a [Truss](https://truss.baseten.co/) for Mixtral 8x7B Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Mistral 7B Instruct. - -**Warning: This example is only intended for usage on a single A100, changing your resource type for this deployment will result in unsupported behavior** - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten. Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -## Deploying Mixtral-8x7B-Instruct - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mixtral-8x7b-instruct-trt-llm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x7b-instruct-trt-llm` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Mixtral 8x7B Instruct API documentation -This section provides an overview of the Mistral 7B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided instruction. - -### API route: `predict` - -This model is designed for our ChatCompletions endpoint: - -- [ChatCompletions endpoint tutorial](https://www.baseten.co/blog/gpt-vs-mistral-migrate-to-open-source-llms-with-minor-code-changes/) -- [ChatCompletions endpoint reference docs](https://docs.baseten.co/api-reference/openai) - -We expect requests will the following information: - -- ```messages``` (str): The prompt you'd like to complete -- ```max_tokens``` (int, default: 50): The max token count. This includes the number of tokens in your prompt so if this value is less than your prompt, you'll just recieve a truncated version of the prompt. -- ```beam_width``` (int, default:50): The number of beams to compute. This must be 1 for this version of TRT-LLM. Inflight-batching does not support beams > 1. -- ```bad_words_list``` (list, default:[]): A list of words to not include in generated output. -- ```stop_words_list``` (list, default:[]): A list of words to stop generation upon encountering. -- ```repetition_penalty``` (float, defualt: 1.0): A repetition penalty to incentivize not repeating tokens. - -This Truss will stream responses back. Responses will be buffered chunks of text. diff --git a/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md b/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md deleted file mode 100644 index d21ff0eda..000000000 --- a/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Mixtral 8x7B Instruct Truss - -This is a [Truss](https://truss.baseten.co/) for Mixtral 8x7B Instruct. Mixtral 8x7B Instruct parameter language model released by [Mistral AI](https://mistral.ai/). It is a mixture-of-experts (MoE) model. This README will walk you through how to deploy this Truss on Baseten to get your own instance of it. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mixtral-8x7b-instruct-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x7b-instruct-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -You need two A100s to run Mixtral at `fp16`. If you need access to A100s, please [contact us](mailto:support@baseten.co). - -## Mixtral 8x7B Instruct API documentation - -This section provides an overview of the Mixtral 8x7B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The `predict` route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __stream__ (optional, default=False): A boolean determining whether the model should stream a response back. When `True`, the API returns generated text as it becomes available. - -## Example usage - -```sh -truss predict -d '{"prompt": "What is the Mistral wind?"}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "What is the meaning of life? Answer in substantial detail with multiple examples from famous philosophies, religions, and schools of thought.", - "stream": true, - "max_tokens": 4096 - }' --no-buffer -``` diff --git a/mistral/mixtral-8x7b-instruct-vllm/README.md b/mistral/mixtral-8x7b-instruct-vllm/README.md deleted file mode 100644 index d21ff0eda..000000000 --- a/mistral/mixtral-8x7b-instruct-vllm/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Mixtral 8x7B Instruct Truss - -This is a [Truss](https://truss.baseten.co/) for Mixtral 8x7B Instruct. Mixtral 8x7B Instruct parameter language model released by [Mistral AI](https://mistral.ai/). It is a mixture-of-experts (MoE) model. This README will walk you through how to deploy this Truss on Baseten to get your own instance of it. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mixtral-8x7b-instruct-vllm -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `mixtral-8x7b-instruct-vllm` as your working directory, you can deploy the model with: - -```sh -truss push --publish -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -You need two A100s to run Mixtral at `fp16`. If you need access to A100s, please [contact us](mailto:support@baseten.co). - -## Mixtral 8x7B Instruct API documentation - -This section provides an overview of the Mixtral 8x7B Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The `predict` route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __stream__ (optional, default=False): A boolean determining whether the model should stream a response back. When `True`, the API returns generated text as it becomes available. - -## Example usage - -```sh -truss predict -d '{"prompt": "What is the Mistral wind?"}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "What is the meaning of life? Answer in substantial detail with multiple examples from famous philosophies, religions, and schools of thought.", - "stream": true, - "max_tokens": 4096 - }' --no-buffer -``` diff --git a/mistral/pixtral-12b/README.md b/mistral/pixtral-12b/README.md deleted file mode 100644 index 5f6420ff6..000000000 --- a/mistral/pixtral-12b/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Pixtral 12B Truss - -This is a [Truss](https://truss.baseten.co/) for Pixtral 12B. Pixtral 12B parameter language model released by [Mistral AI](https://mistral.ai/) and is a multimodal (text + vision) LLM. This README will walk you through how to deploy this Truss on Baseten to get your own instance of it. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd mistral/pixtral-12b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. Accept the terms of service of the Pixtral model [here](https://huggingface.co/mistralai/Pixtral-12B-2409). -4. Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). -5. Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. Note that you will *not* be able to successfully deploy Pixtral without doing this. - -With `pixtral-12b` as your working directory, you can deploy the model with: - -```sh -truss push --publish --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -You need one A100 to run Pixtral at `bf16`. - -## Pixtral 12B API documentation - -This section provides an overview of the Pixtral 12B API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The `predict` route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __messages__: The input in OpenAI API format (see below for examples) -- __stream__ (optional, default=False): A boolean determining whether the model should stream a response back. When `True`, the API returns generated text as it becomes available. -- __max_tokens__ (optional, default=512): Maximum number of tokens to generate -- __temperature__ (optional, default=0.7): A float between 0 and 1. Higher values means the generated output is more random while lower values means the generated output is more determenistic - -## Example usage - -```sh -truss predict -d '{"messages": [{"role": "user", "content": "Tell me about yourself"}]}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What type of animal is this? Answer in French only" - }, - { - "type": "image_url", - "image_url": { - "url": "https://vetmed.illinois.edu/wp-content/uploads/2021/04/pc-keller-hedgehog.jpg" - } - } - ] - } - ], - "stream": true, - "max_tokens": 64, - "temperature": 0.2 - }' --no-buffer -``` diff --git a/multiprocessing/README.md b/multiprocessing/README.md deleted file mode 100644 index 359dd10dc..000000000 --- a/multiprocessing/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Using Multiprocessing for pre- or post-processing steps - -Truss server executes the steps `preprocssing`, `predict` and `postprocess` (or a sub set) using python asyncio. This allows concurrency - and therefore higher throughput - while waiting for I/O-bound tasks (if asyncio-supporting libraries are used e.g. fetching data with async HTTP requrests). -However, it does not increase throughput, if the work in any of those steps is CPU-bound (either by blocking I/O functions or compute intense pre/post-processing operations). - -Specifically, if pre/post-processing is CPU-bound, it can lead to underutilization of the server's GPU resources and suboptimal throughput. -In this case, simply increasing the number server replicas is not efficient, because it would likewise increase the GPU footprint. - -A possible solution is to move the CPU work into separate processes that do not block the main server from using the GPU for *different* requests in the meantime. - -This dummy model illustrates such a setup. To adopt it, the overall inference pipeline must be structured such that most of the CPU work goes into the pre- or post-processing methods and the `predict` method contains mostly GPU work. - - -## Comments on the example code: -* Multiprocessing only demonstrated for `preprocessing`, for `postprocessing` the same pattern (with same process pool) can be used. -* Multiprocessing depends on pickling the function to run (and all its context) to the process pool. If the function depends on unpickleable objects or "large" objects this can be problematic and a different design is needed (see comments in code). -* The worker pool size is determined by the number of CPUs visible to the server. diff --git a/musicgen-large/README.md b/musicgen-large/README.md deleted file mode 100644 index 236e18c70..000000000 --- a/musicgen-large/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# MusicGen Truss - -This repository packages [MusicGen](https://github.com/facebookresearch/audiocraft/) as a [Truss](https://truss.baseten.co). - -MusicGen is a simple and controllable model for music generation developed by Facebook AI Research. - -Utilizing this model for inference can be challenging given the hardware requirements. With Baseten and Truss, inference is dead simple. - -## Deploying MusicGen - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd musicgen-large-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `musicgen-large-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -We found this model runs reasonably fast on A10Gs; you can configure the hardware you'd like in the config.yaml. - -```yaml -resources: - cpu: "3" - memory: 14Gi - use_gpu: true - accelerator: A10G -``` - -## Invoking MusicGen - -MusicGen takes a list of prompts and a duration in seconds. It will generate one clip per prompt and return each clip as a base64 encoded WAV file. - -```sh -truss predict -d '{"prompts": ["happy rock" "energetic EDM", "sad jazz"], "duration": 8}' -``` - -You'll want to pipe your results into a script such as: - -```python -import json -import base64 -import os, sys - -model_output = json.loads(sys.stdin.read()) - -for idx, clip in enumerate(model_output["data"]): - with open(f"clip_{idx}.wav", "wb") as f: - f.write(base64.b64decode(clip)) -``` - -You can also invoke your model via a REST API - -``` -curl -X POST " https://app.baseten.co/models/YOUR_MODEL_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompts": ["happy rock" "energetic EDM", "sad jazz"], "duration": 8 - }' -``` - -## Model sizes - -MusicGen supports four model sizes: - -- `small`: 300M model, text to music only -- `medium`: 1.5B model, text to music only -- `melody`: 1.5B model, text to music and text+melody to music -- `large`: 3.3B model, text to music only - -This truss can been configured to run the large size but you can easily select other versions by changing the `MODEL_SIZE` constant in `model/model.py`. diff --git a/musicgen-melody/README.md b/musicgen-melody/README.md deleted file mode 100644 index 54c132af6..000000000 --- a/musicgen-melody/README.md +++ /dev/null @@ -1,78 +0,0 @@ -# MusicGen Truss - -This repository packages the `melody` model from [MusicGen](https://github.com/facebookresearch/audiocraft/) as a [Truss](https://truss.baseten.co). - -MusicGen is a simple and controllable suite of models for music generation developed by Facebook AI Research. The `melody` model accepts both text and audio to condition it's outputs. - -Utilizing this model for inference can be challenging given the hardware requirements. With Baseten and Truss, inference is dead simple. - -## Deploying MusicGen - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd musicgen-melody-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `musicgen-melody-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -We found this model runs reasonably fast on A10Gs; you can configure the hardware you'd like in the config.yaml. - -```yaml -resources: - cpu: "3" - memory: 14Gi - use_gpu: true - accelerator: A10G -``` - -## Invoking MusicGen - -MusicGen takes a list of prompts and a duration in seconds. You may also, optionally, provide a base64 encoded WAV file as the melody to condition on. It will generate one clip per prompt and return each clip as a base64 encoded WAV file. - -```sh -truss predict -d '{"prompts": ["happy rock" "energetic EDM", "sad jazz"], "melody" : "b64_encoded_melody", "duration": 8}' -``` - -You'll want to pipe your results into a script such as: - -```python -import json -import base64 -import os, sys - -model_output = json.loads(sys.stdin.read()) - -for idx, clip in enumerate(model_output["data"]): - with open(f"clip_{idx}.wav", "wb") as f: - f.write(base64.b64decode(clip)) -``` - -You can also invoke your model via a REST API - -``` -curl -X POST " https://app.baseten.co/models/YOUR_MODEL_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompts": ["happy rock" "energetic EDM", "sad jazz"], - "melody" : "b64_encoded_melody", - "duration": 8 - }' -``` diff --git a/nemotron/llama-3-1-nemotron-70b-instruct/README.md b/nemotron/llama-3-1-nemotron-70b-instruct/README.md deleted file mode 100644 index 33bd03802..000000000 --- a/nemotron/llama-3-1-nemotron-70b-instruct/README.md +++ /dev/null @@ -1,6 +0,0 @@ - -# Llama-3.1-Nemotron-70B-Instruct - -This deployment of Llama-3.1-Nemotron-70B-Instruct uses the TensorRT-LLM Engine Builder. - -For details, see: https://docs.baseten.co/performance/examples/llama-trt diff --git a/nemotron/llama-nemoretriever-colembed-3b-v1/README.md b/nemotron/llama-nemoretriever-colembed-3b-v1/README.md deleted file mode 100644 index 9e8c14b48..000000000 --- a/nemotron/llama-nemoretriever-colembed-3b-v1/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# NVIDIA Llama NemoRetriever ColEmbed 3B V1 - -This is a traditional [Truss](https://truss.baseten.co/) implementation for [NVIDIA's Llama NemoRetriever ColEmbed 3B V1](https://huggingface.co/nvidia/llama-nemoretriever-colembed-3b-v1) cross-modal embedding model. This implementation uses the standard `load()` and `predict()` pattern with a `model.py` file. - -## Support - -For questions or issues, please refer to: -- [Truss Documentation](https://docs.baseten.co) -- [Model Card](https://huggingface.co/nvidia/llama-nemoretriever-colembed-3b-v1) diff --git a/nsql/README.md b/nsql/README.md deleted file mode 100644 index 74d2b5f82..000000000 --- a/nsql/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# NSQL Truss - -This is a [Truss](https://truss.baseten.co/) for [Number Station](https://www.numbersstation.ai/)'s 350M parameter NSQL model. NSQL is a text-to-SQL foundation model, enabling users to query their databases using natual language. There are also 2B and 6B NSQL variants available, which you can alternatively deploy by editing the HuggingFace paths in `model/model.py`. - -This README will walk you through how to deploy this Truss on Baseten to get your own instance of NSQL 350M. - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -## Deploying NSQL - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd nsql-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `nsql-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## NSQL API documentation - -This section provides an overview of the NSQL API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided instruction. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given instruction. It takes several parameters: - -- **schema**: An SQL schema for the table you want to query. You can provide multiple schemas as a single string. -- **query**: A natural language query over the provided database schemas. - -## Example usage - -You can use the `baseten` model package to invoke your model from Python - -``` -import baseten -# You can retrieve your deployed model ID from the UI -model = baseten.deployed_model_version_id('YOUR_MODEL_ID') - -schema = """CREATE TABLE stadium ( - stadium_id number, - location text, - name text, - capacity number, - highest number, - lowest number, - average number -) - -CREATE TABLE singer ( - singer_id number, - name text, - country text, - song_name text, - song_release_year text, - age number, - is_male others -) - -CREATE TABLE concert ( - concert_id number, - concert_name text, - theme text, - stadium_id text, - year text -) - -CREATE TABLE singer_in_concert ( - concert_id number, - singer_id text -)""" - -request = { - "schema": schema, - "query": "What is the maximum, the average, and the minimum capacity of stadiums?" -} - -response = model.predict(request) -``` diff --git a/openai/gpt-oss-120b/README.md b/openai/gpt-oss-120b/README.md deleted file mode 100644 index b960a93d6..000000000 --- a/openai/gpt-oss-120b/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# GPT OSS 120B with BISv2 β€” High-Throughput Template - -GPT OSS 120B is OpenAI's open source model designed for powerful reasoning, agentic tasks and other developer use cases. It uses their open source response format, Harmony. - -This directory contains a **[Truss](https://truss.baseten.co/)** template for deploying **GPT OSS 120B** with **Baseten Inference Stack v2 (TensorRT-LLM + PyTorch backend)** on 4 H100 GPUs. This truss fully abstracts OpenAI's harmony response format, so everything works outside of the box. You can simply use it like a regular OpenAI compatible server. This stack maximizes both inference and throughput. - ---- - -# Requirements - -`truss==0.10.5` - -You also need the file in data, which downloads GPT's harmony encoding ahead of time, because once deployed, the deployment will be unable to download from internet. - -The environment variable `TIKTOKEN_RS_CACHE_DIR: /app/data` in `config.yaml` points `openai_harmony` to the local encoding file. See this(discussion)[https://huggingface.co/openai/gpt-oss-120b/discussions/39] for details. - ---- - - -## Core TRT-LLM `runtime` parameters - -| Property (YAML path) | Value | Why it matters | -| --------------------- | -------------------- | -------------- | -| `tensor_parallel_size`| **4** | Shards every weight matrix across the 2 H100s | -| `moe_expert_parallel_size` | **4** | Shards each expert across 2 H100s | -| `max_batch_size` | **64** | Up to 64 concurrent requests per forward pass | -| `max_seq_len` | **98304** | 98304 token context length | -| `enable_chunked_prefill` | `true` | Chunks long prompts to reduce memory usage | -| `max_num_tokens` | **8192** | Upper limit on total tokens per chunk | -| `served_model_name` | `openai/gpt-oss-120b` | `model: openai/gpt-oss-120b` to call this model in OpenAI Compatible server | - ---- - -## Important Advanced **`runtime.patch_kwargs`** parameters - -These map 1-to-1 to TensorRT-LLM flags for extra performance tuning. - -| Property (YAML path) | Value / Setting | Effect | -| --------------------------------------- | --------------- | ------ | -| `cuda_graph_config.enable_padding` | `true` | Pad to fixed shape so one CUDA Graph is reused every step | -| `kv_cache_config.free_gpu_memory_fraction` | **0.8** | 80 % of post-load VRAM reserved for paged KV-cache | -| `kv_cache_config.enable_block_reuse` | `true` | Identical prefixes share cache blocks β†’ faster TTFT | -| `kv_cache_config.enable_block_reuse` | `true` | Identical prefixes share cache blocks β†’ faster TTFT | -| `chat_processor` | `harmony` | GPT OSS uses Harmony response format | - ---- - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd openai/gpt-oss-120b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `openai/gpt-oss-120b` as your working directory, you can deploy the model with: - -```sh -truss push --trusted --publish -``` - -Paste your Baseten API key if prompted. Also ensure the `hf_access_token` secret is properly setup in your Baseten Account to access this model. - -**Note**: TensorRT-LLM with PyTorch Backend will only work under a Baseten production deployment - -For more information, refer to the [Truss documentation](https://docs.baseten.co/performance/engine-builder-overview). diff --git a/openai/gpt-oss-20b/README.md b/openai/gpt-oss-20b/README.md deleted file mode 100644 index 4ef0a85c2..000000000 --- a/openai/gpt-oss-20b/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# GPT OSS 20B with BISv2 β€” High-Throughput Template - -GPT OSS 20B is OpenAI's open source model designed for powerful reasoning, agentic tasks and other developer use cases. It uses their open source response format, Harmony. - -This directory contains a **[Truss](https://truss.baseten.co/)** template for deploying **GPT OSS 20B** with **Baseten Inference Stack v2 (TensorRT-LLM + PyTorch backend)** on 4 H100 GPUs. This truss fully abstracts OpenAI's harmony response format, so everything works outside of the box. You can simply use it like a regular OpenAI compatible server. This stack maximizes both inference and throughput. - ---- - -# Requirements - -`truss==0.10.5` - -You also need the file in data, which downloads GPT's harmony encoding ahead of time, because once deployed, the deployment will be unable to download from internet. - -The environment variable `TIKTOKEN_RS_CACHE_DIR: /app/data` in `config.yaml` points `openai_harmony` to the local encoding file. See this(discussion)[https://huggingface.co/openai/gpt-oss-20b/discussions/39] for details. - ---- - - -## Core TRT-LLM `runtime` parameters - -| Property (YAML path) | Value | Why it matters | -| --------------------- | -------------------- | -------------- | -| `tensor_parallel_size`| **4** | Shards every weight matrix across the 2 H100s | -| `moe_expert_parallel_size` | **4** | Shards each expert across 2 H100s | -| `max_batch_size` | **64** | Up to 64 concurrent requests per forward pass | -| `max_seq_len` | **98304** | 98304 token context length | -| `enable_chunked_prefill` | `true` | Chunks long prompts to reduce memory usage | -| `max_num_tokens` | **8192** | Upper limit on total tokens per chunk | -| `served_model_name` | `openai/gpt-oss-20b` | `model: openai/gpt-oss-20b` to call this model in OpenAI Compatible server | - ---- - -## Important Advanced **`runtime.patch_kwargs`** parameters - -These map 1-to-1 to TensorRT-LLM flags for extra performance tuning. - -| Property (YAML path) | Value / Setting | Effect | -| --------------------------------------- | --------------- | ------ | -| `cuda_graph_config.enable_padding` | `true` | Pad to fixed shape so one CUDA Graph is reused every step | -| `kv_cache_config.free_gpu_memory_fraction` | **0.8** | 80 % of post-load VRAM reserved for paged KV-cache | -| `kv_cache_config.enable_block_reuse` | `true` | Identical prefixes share cache blocks β†’ faster TTFT | -| `kv_cache_config.enable_block_reuse` | `true` | Identical prefixes share cache blocks β†’ faster TTFT | -| `chat_processor` | `harmony` | GPT OSS uses Harmony response format | - ---- - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd openai/gpt-oss-20b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `openai/gpt-oss-20b` as your working directory, you can deploy the model with: - -```sh -truss push --trusted --publish -``` - -Paste your Baseten API key if prompted. Also ensure the `hf_access_token` secret is properly setup in your Baseten Account to access this model. - -**Note**: TensorRT-LLM with PyTorch Backend will only work under a Baseten production deployment - -For more information, refer to the [Truss documentation](https://docs.baseten.co/performance/engine-builder-overview). diff --git a/optimized/README.md b/optimized/README.md new file mode 100644 index 000000000..3a20de593 --- /dev/null +++ b/optimized/README.md @@ -0,0 +1,22 @@ +# Optimized Models + +Production-grade, TensorRT-LLM optimized model configurations autogenerated by `_internal/templating/generate_templates.py`. These templates are not intended to be edited by hand. + +| Directory | Models | Description | +|-----------|--------|-------------| +| [briton](briton/) | 33 | Baseten Runtime Optimized Networks -- TRT-LLM configs for Llama, Qwen, Gemma, DeepSeek, Mistral, Phi, and Falcon models with FP4/FP8 quantization and speculative decoding variants | +| [bisv2](bisv2/) | 11 | Baseten Inference Server v2 -- optimized serving configs for Llama, Qwen, DeepSeek, and NVIDIA models with FP4/FP8 quantization | + +These configurations are generated from templates. To regenerate them, run: + +```bash +python _internal/templating/generate_templates.py +``` + +## Deploying + +Each optimized model can be deployed to Baseten with: + +```bash +truss push +``` diff --git a/optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md b/optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md new file mode 100644 index 000000000..1dbd604af --- /dev/null +++ b/optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4/README.md @@ -0,0 +1,56 @@ +# BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4-truss-example + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4_kv** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4/config.yaml b/optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4/config.yaml similarity index 89% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4/config.yaml rename to optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4/config.yaml index b9d714cee..e973e54cb 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-deepseek-ai-deepseek-r1-distill-llama-70b-fp4/config.yaml +++ b/optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4/config.yaml @@ -1,3 +1,4 @@ +description: "DeepSeek-R1-Distill-Llama-70B optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md new file mode 100644 index 000000000..949dec9a0 --- /dev/null +++ b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/README.md @@ -0,0 +1,56 @@ +# BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only-truss-example + +Deploy [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 MLP ONLY | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.2-3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.2-3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4_mlp_only** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/config.yaml b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/config.yaml similarity index 90% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/config.yaml rename to optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/config.yaml index 8f9a8bc3e..0ec59fab9 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/config.yaml +++ b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only/config.yaml @@ -1,3 +1,4 @@ +description: "Llama-3.2-3B-Instruct optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8/README.md b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8/README.md new file mode 100644 index 000000000..275a5d4a0 --- /dev/null +++ b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8/README.md @@ -0,0 +1,56 @@ +# BISV2-meta-llama-llama-3.2-3b-instruct-fp8-truss-example + +Deploy [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.2-3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.2-3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8/config.yaml b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8/config.yaml similarity index 90% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8/config.yaml rename to optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8/config.yaml index 4695f8ba1..fdfff00a8 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.2-3b-instruct-fp8/config.yaml +++ b/optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8/config.yaml @@ -1,3 +1,4 @@ +description: "Llama-3.2-3B-Instruct optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4/README.md b/optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4/README.md new file mode 100644 index 000000000..8c071a6d4 --- /dev/null +++ b/optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4/README.md @@ -0,0 +1,56 @@ +# BISV2-meta-llama-llama-3.3-70b-instruct-fp4-truss-example + +Deploy [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.3-70B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4/config.yaml b/optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4/config.yaml similarity index 90% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4/config.yaml rename to optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4/config.yaml index 9d864be6d..7310a7367 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-meta-llama-llama-3.3-70b-instruct-fp4/config.yaml +++ b/optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4/config.yaml @@ -1,3 +1,4 @@ +description: "Llama-3.3-70B-Instruct optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4/README.md b/optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4/README.md new file mode 100644 index 000000000..f0d209789 --- /dev/null +++ b/optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4/README.md @@ -0,0 +1,56 @@ +# BISV2-nvidia-llama-3.1-8b-instruct-fp4-truss-example + +Deploy [nvidia/Llama-3.1-8B-Instruct-FP4](https://huggingface.co/nvidia/Llama-3.1-8B-Instruct-FP4) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/Llama-3.1-8B-Instruct-FP4](https://huggingface.co/nvidia/Llama-3.1-8B-Instruct-FP4) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="nvidia/Llama-3.1-8B-Instruct-FP4", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "nvidia/Llama-3.1-8B-Instruct-FP4", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4/config.yaml b/optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4/config.yaml similarity index 90% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4/config.yaml rename to optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4/config.yaml index ff8e1215d..47a311760 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-llama-3.1-8b-instruct-fp4/config.yaml +++ b/optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4/config.yaml @@ -1,3 +1,4 @@ +description: "Llama-3.1-8B-Instruct-FP4 optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/nvidia-qwen3-30b-a3b-fp4/README.md b/optimized/bisv2/nvidia-qwen3-30b-a3b-fp4/README.md new file mode 100644 index 000000000..b6507308c --- /dev/null +++ b/optimized/bisv2/nvidia-qwen3-30b-a3b-fp4/README.md @@ -0,0 +1,56 @@ +# BISV2-nvidia-qwen3-30b-a3b-fp4-truss-example + +Deploy [nvidia/Qwen3-30B-A3B-FP4](https://huggingface.co/nvidia/Qwen3-30B-A3B-FP4) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/Qwen3-30B-A3B-FP4](https://huggingface.co/nvidia/Qwen3-30B-A3B-FP4) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="nvidia/Qwen3-30B-A3B-FP4", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "nvidia/Qwen3-30B-A3B-FP4", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4/config.yaml b/optimized/bisv2/nvidia-qwen3-30b-a3b-fp4/config.yaml similarity index 90% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4/config.yaml rename to optimized/bisv2/nvidia-qwen3-30b-a3b-fp4/config.yaml index a4f95eb8d..5e335790f 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-30b-a3b-fp4/config.yaml +++ b/optimized/bisv2/nvidia-qwen3-30b-a3b-fp4/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen3-30B-A3B-FP4 optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/nvidia-qwen3-8b-fp4/README.md b/optimized/bisv2/nvidia-qwen3-8b-fp4/README.md new file mode 100644 index 000000000..7ffc7a616 --- /dev/null +++ b/optimized/bisv2/nvidia-qwen3-8b-fp4/README.md @@ -0,0 +1,56 @@ +# BISV2-nvidia-qwen3-8b-fp4-truss-example + +Deploy [nvidia/Qwen3-8B-FP4](https://huggingface.co/nvidia/Qwen3-8B-FP4) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [nvidia/Qwen3-8B-FP4](https://huggingface.co/nvidia/Qwen3-8B-FP4) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="nvidia/Qwen3-8B-FP4", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "nvidia/Qwen3-8B-FP4", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4/config.yaml b/optimized/bisv2/nvidia-qwen3-8b-fp4/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4/config.yaml rename to optimized/bisv2/nvidia-qwen3-8b-fp4/config.yaml index 779a9d144..0ec18889b 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-nvidia-qwen3-8b-fp4/config.yaml +++ b/optimized/bisv2/nvidia-qwen3-8b-fp4/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen3-8B-FP4 optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4/README.md b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4/README.md new file mode 100644 index 000000000..89769337c --- /dev/null +++ b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4/README.md @@ -0,0 +1,56 @@ +# BISV2-qwen-qwen2.5-coder-7b-instruct-fp4-truss-example + +Deploy [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Coder-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4/config.yaml b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4/config.yaml rename to optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4/config.yaml index 5b2b418d3..179430659 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct-fp4/config.yaml +++ b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen2.5-Coder-7B-Instruct optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct/README.md b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct/README.md new file mode 100644 index 000000000..36404cc09 --- /dev/null +++ b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct/README.md @@ -0,0 +1,56 @@ +# BISV2-qwen-qwen2.5-coder-7b-instruct-truss-example + +Deploy [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Coder-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct/config.yaml b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct/config.yaml rename to optimized/bisv2/qwen-qwen2.5-coder-7b-instruct/config.yaml index fc22eaf71..d987b764b 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen2.5-coder-7b-instruct/config.yaml +++ b/optimized/bisv2/qwen-qwen2.5-coder-7b-instruct/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen2.5-Coder-7B-Instruct optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/qwen-qwen3-32b-fp4/README.md b/optimized/bisv2/qwen-qwen3-32b-fp4/README.md new file mode 100644 index 000000000..a164b7149 --- /dev/null +++ b/optimized/bisv2/qwen-qwen3-32b-fp4/README.md @@ -0,0 +1,56 @@ +# BISV2-qwen-qwen3-32b-fp4-truss-example + +Deploy [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4_kv** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4/config.yaml b/optimized/bisv2/qwen-qwen3-32b-fp4/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4/config.yaml rename to optimized/bisv2/qwen-qwen3-32b-fp4/config.yaml index d52a470a0..486d1b63d 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-32b-fp4/config.yaml +++ b/optimized/bisv2/qwen-qwen3-32b-fp4/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen3-32B optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/bisv2/qwen-qwen3-4b-fp8/README.md b/optimized/bisv2/qwen-qwen3-4b-fp8/README.md new file mode 100644 index 000000000..3ed75764c --- /dev/null +++ b/optimized/bisv2/qwen-qwen3-4b-fp8/README.md @@ -0,0 +1,56 @@ +# BISV2-qwen-qwen3-4b-fp8-truss-example + +Deploy [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-4B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-4B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8/config.yaml b/optimized/bisv2/qwen-qwen3-4b-fp8/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8/config.yaml rename to optimized/bisv2/qwen-qwen3-4b-fp8/config.yaml index be29a840a..6cf0cf1b6 100644 --- a/11-embeddings-reranker-classification-tensorrt/BISV2-qwen-qwen3-4b-fp8/config.yaml +++ b/optimized/bisv2/qwen-qwen3-4b-fp8/config.yaml @@ -1,3 +1,4 @@ +description: "Qwen3-4B optimized with BISv2 (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md b/optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md new file mode 100644 index 000000000..b913cb6e8 --- /dev/null +++ b/optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8-truss-example + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8/config.yaml b/optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8/config.yaml rename to optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8/config.yaml index 296c65dde..1d4a2d4a9 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-llama-70b-fp8/config.yaml +++ b/optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "DeepSeek-R1-Distill-Llama-70B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md b/optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md new file mode 100644 index 000000000..ba0bcc28c --- /dev/null +++ b/optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/README.md @@ -0,0 +1,58 @@ +# Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8-truss-example + +Deploy [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/config.yaml b/optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/config.yaml rename to optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/config.yaml index 43b6dc35d..97627c271 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/config.yaml +++ b/optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "DeepSeek-R1-Distill-Qwen-32B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/google-gemma-3-1b-it/README.md b/optimized/briton/google-gemma-3-1b-it/README.md new file mode 100644 index 000000000..4882f6ef0 --- /dev/null +++ b/optimized/briton/google-gemma-3-1b-it/README.md @@ -0,0 +1,59 @@ +# Briton-google-gemma-3-1b-it-truss-example + +Deploy [unsloth/gemma-3-1b-it](https://huggingface.co/unsloth/gemma-3-1b-it) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [unsloth/gemma-3-1b-it](https://huggingface.co/unsloth/gemma-3-1b-it) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="unsloth/gemma-3-1b-it", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "unsloth/gemma-3-1b-it", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it/config.yaml b/optimized/briton/google-gemma-3-1b-it/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it/config.yaml rename to optimized/briton/google-gemma-3-1b-it/config.yaml index d051babbe..2c35bd3f2 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-1b-it/config.yaml +++ b/optimized/briton/google-gemma-3-1b-it/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "gemma-3-1b-it optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/google-gemma-3-270m-it/README.md b/optimized/briton/google-gemma-3-270m-it/README.md new file mode 100644 index 000000000..3d0f1c0ef --- /dev/null +++ b/optimized/briton/google-gemma-3-270m-it/README.md @@ -0,0 +1,59 @@ +# Briton-google-gemma-3-270m-it-truss-example + +Deploy [google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="google/gemma-3-270m-it", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "google/gemma-3-270m-it", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it/config.yaml b/optimized/briton/google-gemma-3-270m-it/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it/config.yaml rename to optimized/briton/google-gemma-3-270m-it/config.yaml index 5a29f2972..3f674b9b7 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-270m-it/config.yaml +++ b/optimized/briton/google-gemma-3-270m-it/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "gemma-3-270m-it optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/google-gemma-3-27b-it-speculative-lookahead/README.md b/optimized/briton/google-gemma-3-27b-it-speculative-lookahead/README.md new file mode 100644 index 000000000..667a4a56f --- /dev/null +++ b/optimized/briton/google-gemma-3-27b-it-speculative-lookahead/README.md @@ -0,0 +1,60 @@ +# Briton-google-gemma-3-27b-it-speculative-lookahead-truss-example + +Deploy [baseten/gemma-3-27b-causallm-it](https://huggingface.co/baseten/gemma-3-27b-causallm-it) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [baseten/gemma-3-27b-causallm-it](https://huggingface.co/baseten/gemma-3-27b-causallm-it) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="baseten/gemma-3-27b-causallm-it", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "baseten/gemma-3-27b-causallm-it", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead/config.yaml b/optimized/briton/google-gemma-3-27b-it-speculative-lookahead/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead/config.yaml rename to optimized/briton/google-gemma-3-27b-it-speculative-lookahead/config.yaml index 72a8349b9..e33398c36 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it-speculative-lookahead/config.yaml +++ b/optimized/briton/google-gemma-3-27b-it-speculative-lookahead/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "gemma-3-27b-causallm-it optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/google-gemma-3-27b-it/README.md b/optimized/briton/google-gemma-3-27b-it/README.md new file mode 100644 index 000000000..7d92cb183 --- /dev/null +++ b/optimized/briton/google-gemma-3-27b-it/README.md @@ -0,0 +1,59 @@ +# Briton-google-gemma-3-27b-it-truss-example + +Deploy [baseten/gemma-3-27b-causallm-it](https://huggingface.co/baseten/gemma-3-27b-causallm-it) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [baseten/gemma-3-27b-causallm-it](https://huggingface.co/baseten/gemma-3-27b-causallm-it) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="baseten/gemma-3-27b-causallm-it", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "baseten/gemma-3-27b-causallm-it", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it/config.yaml b/optimized/briton/google-gemma-3-27b-it/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it/config.yaml rename to optimized/briton/google-gemma-3-27b-it/config.yaml index c9666a8f9..a315df80d 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-google-gemma-3-27b-it/config.yaml +++ b/optimized/briton/google-gemma-3-27b-it/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "gemma-3-27b-causallm-it optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.1-405b-fp8/README.md b/optimized/briton/meta-llama-llama-3.1-405b-fp8/README.md new file mode 100644 index 000000000..05a97c571 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.1-405b-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-meta-llama-llama-3.1-405b-fp8-truss-example + +Deploy [meta-llama/Llama-3.1-405B](https://huggingface.co/meta-llama/Llama-3.1-405B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-405B](https://huggingface.co/meta-llama/Llama-3.1-405B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:8 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.1-405B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-405B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **8** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8/config.yaml b/optimized/briton/meta-llama-llama-3.1-405b-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8/config.yaml rename to optimized/briton/meta-llama-llama-3.1-405b-fp8/config.yaml index 010a4e7f6..62f208929 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-405b-fp8/config.yaml +++ b/optimized/briton/meta-llama-llama-3.1-405b-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.1-405B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md b/optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md new file mode 100644 index 000000000..67b77b5c4 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8-truss-example + +Deploy [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.1-8B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml b/optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml rename to optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml index 85829cea5..574d56993 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml +++ b/optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.1-8B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8/README.md b/optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8/README.md new file mode 100644 index 000000000..ddcc09767 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-meta-llama-llama-3.2-1b-instruct-fp8-truss-example + +Deploy [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.2-1B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.2-1B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8/config.yaml b/optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8/config.yaml rename to optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8/config.yaml index 9006e0ee7..d6ae88c29 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-1b-instruct-fp8/config.yaml +++ b/optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.2-1B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md b/optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md new file mode 100644 index 000000000..0bf9f6118 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8-truss-example + +Deploy [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.2-3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.2-3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/config.yaml b/optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/config.yaml rename to optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/config.yaml index 47a5b3b1c..e5c656fce 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/config.yaml +++ b/optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.2-3B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8/README.md b/optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8/README.md new file mode 100644 index 000000000..88ebaa661 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-meta-llama-llama-3.2-3b-instruct-fp8-truss-example + +Deploy [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.2-3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.2-3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8/config.yaml b/optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8/config.yaml rename to optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8/config.yaml index 8d395a474..8f2ae1078 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct-fp8/config.yaml +++ b/optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.2-3B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.2-3b-instruct/README.md b/optimized/briton/meta-llama-llama-3.2-3b-instruct/README.md new file mode 100644 index 000000000..c27af5c18 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.2-3b-instruct/README.md @@ -0,0 +1,58 @@ +# Briton-meta-llama-llama-3.2-3b-instruct-truss-example + +Deploy [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100_40GB | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.2-3B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.2-3B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct/config.yaml b/optimized/briton/meta-llama-llama-3.2-3b-instruct/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct/config.yaml rename to optimized/briton/meta-llama-llama-3.2-3b-instruct/config.yaml index dcfb1d8a9..da4b58628 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.2-3b-instruct/config.yaml +++ b/optimized/briton/meta-llama-llama-3.2-3b-instruct/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.2-3B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4/README.md b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4/README.md new file mode 100644 index 000000000..18eba6144 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4/README.md @@ -0,0 +1,58 @@ +# Briton-meta-llama-llama-3.3-70b-instruct-fp4-truss-example + +Deploy [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.3-70B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4** +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4/config.yaml b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4/config.yaml rename to optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4/config.yaml index da463d4a4..ca02362a8 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp4/config.yaml +++ b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.3-70B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8/README.md b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8/README.md new file mode 100644 index 000000000..4669284ab --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-meta-llama-llama-3.3-70b-instruct-fp8-truss-example + +Deploy [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.3-70B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8/config.yaml b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8/config.yaml rename to optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8/config.yaml index 215dd074c..108a5a224 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-fp8/config.yaml +++ b/optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.3-70B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md b/optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md new file mode 100644 index 000000000..cadd22884 --- /dev/null +++ b/optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8-truss-example + +Deploy [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:4 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Llama-3.3-70B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **4** GPUs +- Max sequence length: **131,072** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8/config.yaml b/optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8/config.yaml rename to optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8/config.yaml index be35731dc..6abfddd53 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-meta-llama-llama-3.3-70b-instruct-tp4-fp8/config.yaml +++ b/optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Llama-3.3-70B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/microsoft-phi-4-fp8/README.md b/optimized/briton/microsoft-phi-4-fp8/README.md new file mode 100644 index 000000000..c68d9a9f9 --- /dev/null +++ b/optimized/briton/microsoft-phi-4-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-microsoft-phi-4-fp8-truss-example + +Deploy [unsloth/phi-4](https://huggingface.co/unsloth/phi-4) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [unsloth/phi-4](https://huggingface.co/unsloth/phi-4) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | L4:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="unsloth/phi-4", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "unsloth/phi-4", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **16,384** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8/config.yaml b/optimized/briton/microsoft-phi-4-fp8/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8/config.yaml rename to optimized/briton/microsoft-phi-4-fp8/config.yaml index 972fad2c8..51f8bc64d 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-microsoft-phi-4-fp8/config.yaml +++ b/optimized/briton/microsoft-phi-4-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "phi-4 optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/mistralai-mistral-7b-instruct-v0.3/README.md b/optimized/briton/mistralai-mistral-7b-instruct-v0.3/README.md new file mode 100644 index 000000000..580d57d4f --- /dev/null +++ b/optimized/briton/mistralai-mistral-7b-instruct-v0.3/README.md @@ -0,0 +1,59 @@ +# Briton-mistralai-mistral-7b-instruct-v0.3-truss-example + +Deploy [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | A10G:2 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mistral-7B-Instruct-v0.3", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mistral-7B-Instruct-v0.3", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Tensor parallelism: **2** GPUs +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3/config.yaml b/optimized/briton/mistralai-mistral-7b-instruct-v0.3/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3/config.yaml rename to optimized/briton/mistralai-mistral-7b-instruct-v0.3/config.yaml index eec4a13d8..9827dd88e 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-7b-instruct-v0.3/config.yaml +++ b/optimized/briton/mistralai-mistral-7b-instruct-v0.3/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Mistral-7B-Instruct-v0.3 optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8/README.md b/optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8/README.md new file mode 100644 index 000000000..93a8cd2c6 --- /dev/null +++ b/optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-mistralai-mistral-small-24b-instruct-2501-fp8-truss-example + +Deploy [mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="mistralai/Mistral-Small-24B-Instruct-2501", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "mistralai/Mistral-Small-24B-Instruct-2501", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8/config.yaml b/optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8/config.yaml rename to optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8/config.yaml index 97449aeb5..3df60f735 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-mistralai-mistral-small-24b-instruct-2501-fp8/config.yaml +++ b/optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Mistral-Small-24B-Instruct-2501 optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8/README.md b/optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8/README.md new file mode 100644 index 000000000..48922e097 --- /dev/null +++ b/optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen2.5-72b-instruct-tp2-fp8-truss-example + +Deploy [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-72B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-72B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Tensor parallelism: **2** GPUs +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8/config.yaml b/optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8/config.yaml rename to optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8/config.yaml index f07e9d02f..09c8b007d 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-72b-instruct-tp2-fp8/config.yaml +++ b/optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen2.5-72B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md b/optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md new file mode 100644 index 000000000..6d35b7298 --- /dev/null +++ b/optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8-truss-example + +Deploy [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml b/optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml rename to optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml index 7438254e5..5bee03956 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml +++ b/optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen2.5-7B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md new file mode 100644 index 000000000..22b2291a6 --- /dev/null +++ b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/README.md @@ -0,0 +1,58 @@ +# Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only-truss-example + +Deploy [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 MLP ONLY | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Coder-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4_mlp_only** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/config.yaml b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/config.yaml rename to optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/config.yaml index f7b8b2cfc..961e00329 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/config.yaml +++ b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen2.5-Coder-7B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md new file mode 100644 index 000000000..17b7b953b --- /dev/null +++ b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8-truss-example + +Deploy [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen2.5-Coder-7B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/config.yaml b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/config.yaml similarity index 94% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/config.yaml rename to optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/config.yaml index 40f515706..e3e3956d0 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/config.yaml +++ b/optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen2.5-Coder-7B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md b/optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md new file mode 100644 index 000000000..a391a012d --- /dev/null +++ b/optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8/README.md @@ -0,0 +1,62 @@ +# Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8-truss-example + +Deploy [Qwen/Qwen3-235B-A22B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-235B-A22B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:8 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-235B-A22B-Instruct-2507", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-235B-A22B-Instruct-2507", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **8** GPUs +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **262,144** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8/config.yaml b/optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8/config.yaml rename to optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8/config.yaml index 3e61aa0ce..314fd33ee 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-235b-a22b-instruct-2507-fp8/config.yaml +++ b/optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-235B-A22B-Instruct-2507 optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-30b-a3b-fp8/README.md b/optimized/briton/qwen-qwen3-30b-a3b-fp8/README.md new file mode 100644 index 000000000..161735e8a --- /dev/null +++ b/optimized/briton/qwen-qwen3-30b-a3b-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen3-30b-a3b-fp8-truss-example + +Deploy [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-30B-A3B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-30B-A3B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8/config.yaml b/optimized/briton/qwen-qwen3-30b-a3b-fp8/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8/config.yaml rename to optimized/briton/qwen-qwen3-30b-a3b-fp8/config.yaml index 9a2e2a2b9..c26c05360 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-fp8/config.yaml +++ b/optimized/briton/qwen-qwen3-30b-a3b-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-30B-A3B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md b/optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md new file mode 100644 index 000000000..acb19d843 --- /dev/null +++ b/optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8-truss-example + +Deploy [Qwen/Qwen3-30B-A3B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-30B-A3B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-30B-A3B-Instruct-2507", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-30B-A3B-Instruct-2507", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **262,144** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8/config.yaml b/optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8/config.yaml similarity index 91% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8/config.yaml rename to optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8/config.yaml index c767998f0..6f94aa9da 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b-instruct-2507-fp8/config.yaml +++ b/optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-30B-A3B-Instruct-2507 optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-30b-a3b/README.md b/optimized/briton/qwen-qwen3-30b-a3b/README.md new file mode 100644 index 000000000..3932c6a5d --- /dev/null +++ b/optimized/briton/qwen-qwen3-30b-a3b/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen3-30b-a3b-truss-example + +Deploy [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100:2 | +| Quantization | NO QUANT | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-30B-A3B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-30B-A3B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **no_quant** +- Tensor parallelism: **2** GPUs +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b/config.yaml b/optimized/briton/qwen-qwen3-30b-a3b/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b/config.yaml rename to optimized/briton/qwen-qwen3-30b-a3b/config.yaml index 994438d5c..386807b37 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-30b-a3b/config.yaml +++ b/optimized/briton/qwen-qwen3-30b-a3b/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-30B-A3B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-32b-fp4-mlp-only/README.md b/optimized/briton/qwen-qwen3-32b-fp4-mlp-only/README.md new file mode 100644 index 000000000..9686ddbcd --- /dev/null +++ b/optimized/briton/qwen-qwen3-32b-fp4-mlp-only/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen3-32b-fp4-mlp-only-truss-example + +Deploy [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 MLP ONLY | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4_mlp_only** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only/config.yaml b/optimized/briton/qwen-qwen3-32b-fp4-mlp-only/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only/config.yaml rename to optimized/briton/qwen-qwen3-32b-fp4-mlp-only/config.yaml index 5e919d604..5161c5b02 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4-mlp-only/config.yaml +++ b/optimized/briton/qwen-qwen3-32b-fp4-mlp-only/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-32B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-32b-fp4/README.md b/optimized/briton/qwen-qwen3-32b-fp4/README.md new file mode 100644 index 000000000..55ab80362 --- /dev/null +++ b/optimized/briton/qwen-qwen3-32b-fp4/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwen3-32b-fp4-truss-example + +Deploy [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | B200 | +| Quantization | FP4 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp4_kv** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4/config.yaml b/optimized/briton/qwen-qwen3-32b-fp4/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4/config.yaml rename to optimized/briton/qwen-qwen3-32b-fp4/config.yaml index 94709b289..f91afaaf4 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp4/config.yaml +++ b/optimized/briton/qwen-qwen3-32b-fp4/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-32B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-32b-fp8/README.md b/optimized/briton/qwen-qwen3-32b-fp8/README.md new file mode 100644 index 000000000..241c36276 --- /dev/null +++ b/optimized/briton/qwen-qwen3-32b-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-qwen-qwen3-32b-fp8-truss-example + +Deploy [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Batch scheduler policy: **max_utilization** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8/config.yaml b/optimized/briton/qwen-qwen3-32b-fp8/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8/config.yaml rename to optimized/briton/qwen-qwen3-32b-fp8/config.yaml index f99994354..9065d08b8 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-32b-fp8/config.yaml +++ b/optimized/briton/qwen-qwen3-32b-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-32B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwen3-8b-min-latency-fp8/README.md b/optimized/briton/qwen-qwen3-8b-min-latency-fp8/README.md new file mode 100644 index 000000000..4a2f18da3 --- /dev/null +++ b/optimized/briton/qwen-qwen3-8b-min-latency-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-qwen-qwen3-8b-min-latency-fp8-truss-example + +Deploy [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/Qwen3-8B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen3-8B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8/config.yaml b/optimized/briton/qwen-qwen3-8b-min-latency-fp8/config.yaml similarity index 95% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8/config.yaml rename to optimized/briton/qwen-qwen3-8b-min-latency-fp8/config.yaml index e809dd7c4..2d33f24e9 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwen3-8b-min-latency-fp8/config.yaml +++ b/optimized/briton/qwen-qwen3-8b-min-latency-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Qwen3-8B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwq-32b-reasoning-fp8/README.md b/optimized/briton/qwen-qwq-32b-reasoning-fp8/README.md new file mode 100644 index 000000000..7c22b16ef --- /dev/null +++ b/optimized/briton/qwen-qwq-32b-reasoning-fp8/README.md @@ -0,0 +1,58 @@ +# Briton-qwen-qwq-32b-reasoning-fp8-truss-example + +Deploy [Qwen/QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/QwQ-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/QwQ-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8/config.yaml b/optimized/briton/qwen-qwq-32b-reasoning-fp8/config.yaml similarity index 93% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8/config.yaml rename to optimized/briton/qwen-qwq-32b-reasoning-fp8/config.yaml index 8ef63567f..6c4eeac74 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-fp8/config.yaml +++ b/optimized/briton/qwen-qwq-32b-reasoning-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "QwQ-32B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8/README.md b/optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8/README.md new file mode 100644 index 000000000..779fb7e24 --- /dev/null +++ b/optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8/README.md @@ -0,0 +1,59 @@ +# Briton-qwen-qwq-32b-reasoning-with-speculative-fp8-truss-example + +Deploy [Qwen/QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [Qwen/QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | H100 | +| Quantization | FP8 | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="Qwen/QwQ-32B", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/QwQ-32B", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8** +- Speculative decoding: **LOOKAHEAD_DECODING** +- Max sequence length: **40,960** +- Chunked context: **enabled** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8/config.yaml b/optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8/config.yaml similarity index 95% rename from 11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8/config.yaml rename to optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8/config.yaml index 5dfcd2a1a..f7fa10268 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-qwen-qwq-32b-reasoning-with-speculative-fp8/config.yaml +++ b/optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "QwQ-32B optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/optimized/briton/tiiuae-falcon3-10b-instruct-fp8/README.md b/optimized/briton/tiiuae-falcon3-10b-instruct-fp8/README.md new file mode 100644 index 000000000..6441a13d4 --- /dev/null +++ b/optimized/briton/tiiuae-falcon3-10b-instruct-fp8/README.md @@ -0,0 +1,60 @@ +# Briton-tiiuae-falcon3-10b-instruct-fp8-truss-example + +Deploy [tiiuae/Falcon3-10B-Instruct](https://huggingface.co/tiiuae/Falcon3-10B-Instruct) for text generation using a TRT-LLM engine on Baseten. + +| Property | Value | +|----------|-------| +| Model | [tiiuae/Falcon3-10B-Instruct](https://huggingface.co/tiiuae/Falcon3-10B-Instruct) | +| Task | Text generation | +| Engine | TRT-LLM | +| GPU | L4:2 | +| Quantization | FP8 KV | +| OpenAI compatible | Yes | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +This model is OpenAI-compatible. You can use the OpenAI Python client or curl. + +**Python (OpenAI SDK):** + +```python +from openai import OpenAI + +client = OpenAI( + api_key="YOUR_BASETEN_API_KEY", + base_url="https://model-.api.baseten.co/v1", +) + +response = client.chat.completions.create( + model="tiiuae/Falcon3-10B-Instruct", + messages=[{"role": "user", "content": "What is machine learning?"}], + max_tokens=512, +) + +print(response.choices[0].message.content) +``` + +**curl:** + +```sh +curl -X POST https://model-.api.baseten.co/v1/chat/completions \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "tiiuae/Falcon3-10B-Instruct", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 512}' +``` + +## Configuration highlights + +- Quantization: **fp8_kv** +- Tensor parallelism: **2** GPUs +- Max sequence length: **32,768** +- Chunked context: **enabled** +- Plugin: **use_fp8_context_fmha** +- Streaming: **enabled** diff --git a/11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8/config.yaml b/optimized/briton/tiiuae-falcon3-10b-instruct-fp8/config.yaml similarity index 92% rename from 11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8/config.yaml rename to optimized/briton/tiiuae-falcon3-10b-instruct-fp8/config.yaml index 3548da52d..23a260c14 100644 --- a/11-embeddings-reranker-classification-tensorrt/Briton-tiiuae-falcon3-10b-instruct-fp8/config.yaml +++ b/optimized/briton/tiiuae-falcon3-10b-instruct-fp8/config.yaml @@ -1,4 +1,5 @@ # this file was autogenerated by `generate_templates.py` - please do change via template only +description: "Falcon3-10B-Instruct optimized with Briton (TRT-LLM)" model_metadata: example_model_input: max_tokens: 512 diff --git a/orpheus-best-performance/README.md b/orpheus-best-performance/README.md deleted file mode 100644 index 3feae8dff..000000000 --- a/orpheus-best-performance/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Orpheus TTS - -To deploy this performance-optimized implementation of Orpheus TTS, first ensure everything is correct in config.yaml then run: - -``` -pip install --upgrade truss -truss push --publish orpheus-best-performance -``` diff --git a/personaplex-7b-v1/README.md b/personaplex-7b-v1/README.md deleted file mode 100644 index dafe3b0b2..000000000 --- a/personaplex-7b-v1/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# PersonaPlex 7V - Real-time Speech-to-Speech - -This example demonstrates deploying NVIDIA's PersonaPlex 7B v1 model using Baseten's Bring Your Own Image (BYOI) feature with a custom Docker image containing a forked version of PersonaPlex. - -## Overview - -PersonaPlex is a speech-to-speech AI model that enables real-time voice conversations. This deployment: - -1. **Custom Docker Image**: We built a Docker image (`basetenservice/personaplex-7v:fork`) that builds a fork of PersonaPlex from [basetenlabs/personaplex-baseten](https://github.com/basetenlabs/personaplex-baseten) -2. **Baseten BYOI**: The Docker image is deployed using Baseten's bring your own image capability, which allows us to use custom base images with the Baseten platform -3. **WebSocket Protocol**: The model communicates via WebSocket for low-latency bidirectional audio streaming - -The dockerfile is included in this folder. The `config.yaml` brings this image and runs the Moshi server on port 8998. - -## Server Setup - -To deploy PersonaPlex 7V on Baseten, you'll use the Truss CLI to push this folder - -### Requirements - -- Truss CLI installed (`pip install truss`), cd into this directory, and `truss push --publish` -- Baseten account and API key -- Hugging Face account and Read Access Token -``` - -### Deploy the Model - -From the `personaplex-7b-v1` directory, push your Truss: - -```bash -truss push . --byoi --env HUGGINGFACE_TOKEN=$HUGGINGFACE_TOKEN -``` - -This command builds your custom Docker image, passes the Hugging Face token as a runtime environment variable, and deploys your model to Baseten. - -After deployment, your WebSocket server will be available and ready to accept client connections. - -## Client Setup - -### Prerequisites - -The client to connect to the server requires the Opus audio codec and Python dependencies. - -#### macOS - -```bash -brew install opus -``` - -#### Linux (Ubuntu/Debian) - -```bash -sudo apt-get install libopus-dev -``` - -### Install Python Dependencies - -```bash -pip install -r requirements-client.txt -``` - -This installs: -- `websockets` - WebSocket client library -- `numpy` - Array operations -- `sphn` - Opus audio codec Python bindings -- `sounddevice` - Audio I/O - -### Set API Key - -```bash -export BASETEN_API_KEY="your_api_key_here" -``` - -## Running the Client - -```bash -python client.py -``` - -The client will: -1. Connect to the deployed model via WebSocket -2. Start your microphone and speakers -3. Enable real-time voice conversation with the AI - -## WebSocket Protocol - -The communication protocol uses a simple binary format: - -### Initial Handshake - -1. **Client sends config** (first message): JSON configuration - ```json - { - "voice_prompt": "NATF0.pt", - "text_prompt": "You are a helpful assistant.", - "seed": -1 - } - ``` - -2. **Server responds**: Single byte `0x00` to confirm readiness - -### Streaming Messages - -After the handshake, all messages are binary with a single-byte header: - -- **`0x01` + Opus audio data**: Audio frames (sent by both client and server) -- **`0x02` + UTF-8 text**: Text transcriptions/responses (sent by server) - -The first byte of each message indicates the payload type, and the remaining bytes contain the Opus-encoded audio or UTF-8 text. - -## Configuration - -The model is configured in `config.yaml`: -- **Image**: Contains the built model at nvidia/personaplex-7b-v1 -- **Accelerator**: H100 GPU -- **Transport**: WebSocket -- **Secrets**: Requires HuggingFace access token for model download diff --git a/phi/phi-3-mini-128k-instruct/README.md b/phi/phi-3-mini-128k-instruct/README.md deleted file mode 100644 index a8cf1d6b8..000000000 --- a/phi/phi-3-mini-128k-instruct/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Phi-3-Mini-128K-Instruct - -This is a [Truss](https://truss.baseten.co/) for Phi-3-Mini-128K-Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Llama 3 8B. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd phi/phi-3-mini-128k-instruct -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `phi/phi-3-mini-128k-instruct` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Phi-3-Mini-128K-Instruct API documentation - -This section provides an overview of the Phi-3-Mini-128K-Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __messages__: The input text that you want the model to generate a response for. -- __max_tokens__ (optional, default=512): The maximum number of tokens to return, counting input tokens. Maximum of 4096. -- __temperature__ (optional, default=1.0): Controls the randomness of the generated text. Higher values produce more diverse results, while lower values produce more deterministic results. -- __top_p__ (optional, default=0.75): The cumulative probability threshold for token sampling. The model will only consider tokens whose cumulative probability is below this threshold. -- __top_k__ (optional, default=40): The number of top tokens to consider when sampling. The model will only consider the top_k highest-probability tokens. - -The API also supports passing any parameter supported by HuggingFace's `Transformers.generate`. - -## Example usage - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "messages": [{"role": "user", "content": "What even is AGI?"}], - "max_tokens": 256 - }' -``` diff --git a/phi/phi-3-mini-128k-instruct/config.yaml b/phi/phi-3-mini-128k-instruct/config.yaml deleted file mode 100644 index db4324c6b..000000000 --- a/phi/phi-3-mini-128k-instruct/config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_metadata: {} -model_name: Phi-3-Mini-128K-Instruct -python_version: py39 -requirements: - - accelerate - - einops - - transformers==4.40.1 - - torch==2.3.0 -resources: - accelerator: T4 - use_gpu: true -secrets: {} -system_packages: [] diff --git a/phi/phi-3-mini-4k-instruct/README.md b/phi/phi-3-mini-4k-instruct/README.md deleted file mode 100644 index 19d4ddda8..000000000 --- a/phi/phi-3-mini-4k-instruct/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Phi-3-Mini-4K-Instruct - -This is a [Truss](https://truss.baseten.co/) for Phi-3-Mini-4K-Instruct. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Llama 3 8B. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd phi/phi-3-mini-4k-instruct -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `phi/phi-3-mini-4k-instruct` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Phi-3-Mini-4K-Instruct API documentation - -This section provides an overview of the Phi-3-Mini-4K-Instruct API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __messages__: The input text that you want the model to generate a response for. -- __max_tokens__ (optional, default=512): The maximum number of tokens to return, counting input tokens. Maximum of 4096. -- __temperature__ (optional, default=1.0): Controls the randomness of the generated text. Higher values produce more diverse results, while lower values produce more deterministic results. -- __top_p__ (optional, default=0.75): The cumulative probability threshold for token sampling. The model will only consider tokens whose cumulative probability is below this threshold. -- __top_k__ (optional, default=40): The number of top tokens to consider when sampling. The model will only consider the top_k highest-probability tokens. - -The API also supports passing any parameter supported by HuggingFace's `Transformers.generate`. - -## Example usage - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "messages": [{"role": "user", "content": "What even is AGI?"}], - "max_tokens": 256 - }' -``` diff --git a/phi/phi-3-mini-4k-instruct/config.yaml b/phi/phi-3-mini-4k-instruct/config.yaml deleted file mode 100644 index 20f2c30e8..000000000 --- a/phi/phi-3-mini-4k-instruct/config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_metadata: {} -model_name: Phi-3-Mini-4K-Instruct -python_version: py39 -requirements: - - accelerate - - einops - - transformers==4.40.1 - - torch==2.3.0 -resources: - accelerator: T4 - use_gpu: true -secrets: {} -system_packages: [] diff --git a/phi/phi-3.5-mini/README.md b/phi/phi-3.5-mini/README.md deleted file mode 100644 index 87be85f77..000000000 --- a/phi/phi-3.5-mini/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# Phi 3.5 Mini Instruct - -This is a [Truss](https://truss.baseten.co/) example using our general purpose [vLLM Template](https://github.com/basetenlabs/truss-examples/tree/main/vllm) but for Phi-3.5-Mini-instruct, one of the [compatible chat completion models](https://docs.vllm.ai/en/latest/models/supported_models.html). - -> Note: `prefix_caching` is not supported by vLLM for this model, please do not include `enable_prefix_caching` as part of the vllm config in `config.yaml` - - - -## Deploy your Truss - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. With `vllm` as your working directory, you can deploy the model with: - - ```sh - truss push --trusted - ``` - - Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Call your model - -Once your deployment is up, there are [many ways](https://docs.baseten.co/invoke/quickstart) to call your model. - -### curl command - -#### If you are NOT using OpenAI compatible server - -``` -curl -X POST https://model-.api.baseten.co/development/predict \ - -H "Authorization: Api-Key $BASETEN_API_KEY" \ - -d '{"prompt": "what is the meaning of life"}' -``` - - -#### If you are using OpenAI compatible server - -``` -curl -X POST "https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {BASETEN_API_KEY}' \ - -d '{ - "messages": [{"role": "user", "content": "What even is AGI?"}], - "max_tokens": 256 - }' -``` - -To access [production metrics](https://docs.vllm.ai/en/latest/serving/metrics.html) reported by OpenAI compatible server, simply add `metrics: true` to the request. - -``` -curl -X POST "https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {BASETEN_API_KEY}' \ - -d '{ - "metrics": true - }' -``` - -### OpenAI SDK (if you are using OpenAI compatible server) - -``` -from openai import OpenAI -import os - -model_id = "abcd1234" # Replace with your model ID -deployment_id = "4321cbda" # [Optional] Replace with your deployment ID - -client = OpenAI( - api_key=os.environ["BASETEN_API_KEY"], - base_url=f"https://bridge.baseten.co/v1/direct" -) - -response = client.chat.completions.create( - model="microsoft/Phi-3.5-mini-instruct", - messages=[ - {"role": "user", "content": "Who won the world series in 2020?"}, - {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."}, - {"role": "user", "content": "Where was it played?"} - ], - extra_body={ - "baseten": { - "model_id": model_id, - "deployment_id": deployment_id - } - } -) - -print(response.choices[0].message.content) - -``` - -For more information, see [API reference](https://docs.baseten.co/api-reference/openai). - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/piper-tts/README.md b/piper-tts/README.md deleted file mode 100644 index 7bb375fce..000000000 --- a/piper-tts/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Piper TTS Truss - -This repository packages [Piper-TTS](https://github.com/rhasspy/piper) as a [Truss](https://truss.baseten.co/). - -Piper TTS is a generative audio model for text-to-speech generation. This model can be trained on various voices to create realistic output audio. This model has very low latency and is optimized to run on a Raspberry Pi! - -## Deploying Piper TTS - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd piper-tts -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `piper-tts` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - - -## Hardware Requirements -This model does not need a GPU to run. However, the configuration provided with this Truss does have a GPU enabled for extra performance. - -## Invoking the model - -Here are the following inputs for the model: -1. `text`: The text that needs to be converted into speech - -Here is an example of how to invoke this model: - -```python -import base64 -import requests -import os - -def base64_to_wav(base64_string, output_file_path): - binary_data = base64.b64decode(base64_string) - with open(output_file_path, "wb") as wav_file: - wav_file.write(binary_data) - -text = "Listen up, people. Life's a wild ride, and sometimes you gotta grab it by the horns and steer it where you want to go. You can't just sit around waiting for things to happen – you gotta make 'em happen. Yeah, it's gonna get tough, but that's when you dig deep, find that inner badass, and come out swinging. Remember, success ain't handed to you on a silver platter; you gotta snatch it like it owes you money. So, lace up your boots, square those shoulders, and let the world know that you're here to play, and you're playing for keeps" -data = {"text": text} -headers = {"Authorization": f"Api-Key "} -res = requests.post("https://model-.api.baseten.co/development/predict", headers=headers, json=data) -res = res.json() -output = base64_to_wav(res.get('output'), "piper-tts-output.wav") -os.system("open piper-tts-output.wav") -``` - -The output of the model is a base64 string, so you can convert it to a wav file using the `base64_to_wav` function. - -Here is the output from the model using the input above: - -https://github.com/htrivedi99/truss-examples/assets/15642666/d35a9da3-f49c-4820-ab1e-08982d893598 diff --git a/playground-v2-aesthetic/README.md b/playground-v2-aesthetic/README.md deleted file mode 100644 index 15327a4e8..000000000 --- a/playground-v2-aesthetic/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Playground V2 Aesthetic Truss - -This is a [Truss](https://truss.baseten.co/welcome) for [Playground V2 Aesthetic](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) - -Playground v2 is a diffusion-based text-to-image generative model. The model was trained from scratch by the research team at [Playground](https://playground.com). The research team curated 3,000 samples from Midjourney per category to train their model. According to their human preference study, people prefers the output of this model 2.5X more than Stable Diffusion XL. - -Here are some examples of the outputs from this model: - -![aesthetic_images](https://github.com/htrivedi99/truss-examples/assets/15642666/6c3fc815-73ec-4ab1-abab-7723884791f1) - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd playground-v2-aesthetic -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `playground-v2-aesthetic` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### API route: `predict` - -The predict route is the primary method for generating images based on a given prompt. It takes several parameters: - -- `prompt` (required): The input text required for image generation. -- `negative_prompt` (optional): Use this to refine the image generation by discarding unwanted items. -- `scheduler` (optional): This controls which scheduler to use resulting in more image variations. -- `steps` (optional): The number of iterations the model runs through. -- `guidance_scale` (optional): Used to control how closely the image generation follows the prompt. -- `seed` (optional): Random number used to control image variations. - -The output of the model is an image in the form of a base64 string. -Example model output: `{"output": "BASE64-STRING"}` - -## Example usage - -```sh -truss predict -d '{"prompt": "An astronaut snowboarding on an alient planet, highly detailed, 8K"}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "An astronaut snowboarding on an alient planet, highly detailed, 8K" - }' -``` - -You can also use Python to invoke the model: - -``` python -import os -import json -import base64 -import requests - -# Set essential values -model_id = "" -baseten_api_key = "" - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/development/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json={ - "prompt": "a futuristic motorcycle, neon colors, cyberpunk city, detailed, 8K", - "steps": 50 - } -) -# Get output image -res = res.json() -output = json.loads(res)["output"] -image = base64.b64decode(output) -# Save image to file -img_file = open("playground.png", "wb") -img_file.write(image) -img_file.close() -os.system("open playground.png") -``` diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index 4d3793a89..000000000 --- a/poetry.lock +++ /dev/null @@ -1,609 +0,0 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. - -[[package]] -name = "asttokens" -version = "3.0.0" -description = "Annotate AST trees with source code positions" -optional = false -python-versions = ">=3.8" -files = [ - {file = "asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2"}, - {file = "asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7"}, -] - -[package.extras] -astroid = ["astroid (>=2,<4)"] -test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] - -[[package]] -name = "black" -version = "23.12.1" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.8" -files = [ - {file = "black-23.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2"}, - {file = "black-23.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba"}, - {file = "black-23.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0"}, - {file = "black-23.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3"}, - {file = "black-23.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba"}, - {file = "black-23.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b"}, - {file = "black-23.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59"}, - {file = "black-23.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50"}, - {file = "black-23.12.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e"}, - {file = "black-23.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec"}, - {file = "black-23.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e"}, - {file = "black-23.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9"}, - {file = "black-23.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f"}, - {file = "black-23.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d"}, - {file = "black-23.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a"}, - {file = "black-23.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e"}, - {file = "black-23.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055"}, - {file = "black-23.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54"}, - {file = "black-23.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea"}, - {file = "black-23.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2"}, - {file = "black-23.12.1-py3-none-any.whl", hash = "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e"}, - {file = "black-23.12.1.tar.gz", hash = "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -packaging = ">=22.0" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "cfgv" -version = "3.4.0" -description = "Validate configuration and produce human readable error messages." -optional = false -python-versions = ">=3.8" -files = [ - {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, - {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, -] - -[[package]] -name = "click" -version = "8.1.8" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, - {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "decorator" -version = "5.2.1" -description = "Decorators for Humans" -optional = false -python-versions = ">=3.8" -files = [ - {file = "decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a"}, - {file = "decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360"}, -] - -[[package]] -name = "distlib" -version = "0.3.9" -description = "Distribution utilities" -optional = false -python-versions = "*" -files = [ - {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"}, - {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"}, -] - -[[package]] -name = "exceptiongroup" -version = "1.3.0" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, - {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "executing" -version = "2.2.0" -description = "Get the currently executing AST node of a frame, and other information" -optional = false -python-versions = ">=3.8" -files = [ - {file = "executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa"}, - {file = "executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755"}, -] - -[package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] - -[[package]] -name = "filelock" -version = "3.18.0" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.9" -files = [ - {file = "filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de"}, - {file = "filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2"}, -] - -[package.extras] -docs = ["furo (>=2024.8.6)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.6.10)", "diff-cover (>=9.2.1)", "pytest (>=8.3.4)", "pytest-asyncio (>=0.25.2)", "pytest-cov (>=6)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.28.1)"] -typing = ["typing-extensions (>=4.12.2)"] - -[[package]] -name = "identify" -version = "2.6.12" -description = "File identification library for Python" -optional = false -python-versions = ">=3.9" -files = [ - {file = "identify-2.6.12-py2.py3-none-any.whl", hash = "sha256:ad9672d5a72e0d2ff7c5c8809b62dfa60458626352fb0eb7b55e69bdc45334a2"}, - {file = "identify-2.6.12.tar.gz", hash = "sha256:d8de45749f1efb108badef65ee8386f0f7bb19a7f26185f74de6367bffbaf0e6"}, -] - -[package.extras] -license = ["ukkonen"] - -[[package]] -name = "ipython" -version = "8.18.1" -description = "IPython: Productive Interactive Computing" -optional = false -python-versions = ">=3.9" -files = [ - {file = "ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397"}, - {file = "ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -decorator = "*" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} -jedi = ">=0.16" -matplotlib-inline = "*" -pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} -prompt-toolkit = ">=3.0.41,<3.1.0" -pygments = ">=2.4.0" -stack-data = "*" -traitlets = ">=5" -typing-extensions = {version = "*", markers = "python_version < \"3.10\""} - -[package.extras] -all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] -black = ["black"] -doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] -kernel = ["ipykernel"] -nbconvert = ["nbconvert"] -nbformat = ["nbformat"] -notebook = ["ipywidgets", "notebook"] -parallel = ["ipyparallel"] -qtconsole = ["qtconsole"] -test = ["pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath"] -test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath", "trio"] - -[[package]] -name = "isort" -version = "5.13.2" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, - {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, -] - -[package.extras] -colors = ["colorama (>=0.4.6)"] - -[[package]] -name = "jedi" -version = "0.19.2" -description = "An autocompletion tool for Python that can be used for text editors." -optional = false -python-versions = ">=3.6" -files = [ - {file = "jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9"}, - {file = "jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0"}, -] - -[package.dependencies] -parso = ">=0.8.4,<0.9.0" - -[package.extras] -docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] -qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] -testing = ["Django", "attrs", "colorama", "docopt", "pytest (<9.0.0)"] - -[[package]] -name = "matplotlib-inline" -version = "0.1.7" -description = "Inline Matplotlib backend for Jupyter" -optional = false -python-versions = ">=3.8" -files = [ - {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"}, - {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"}, -] - -[package.dependencies] -traitlets = "*" - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.8" -files = [ - {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, - {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, -] - -[[package]] -name = "nodeenv" -version = "1.9.1" -description = "Node.js virtual environment builder" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, - {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, -] - -[[package]] -name = "packaging" -version = "25.0" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, - {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, -] - -[[package]] -name = "parso" -version = "0.8.4" -description = "A Python Parser" -optional = false -python-versions = ">=3.6" -files = [ - {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, - {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, -] - -[package.extras] -qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] -testing = ["docopt", "pytest"] - -[[package]] -name = "pathspec" -version = "0.12.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, - {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, -] - -[[package]] -name = "pexpect" -version = "4.9.0" -description = "Pexpect allows easy control of interactive console applications." -optional = false -python-versions = "*" -files = [ - {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, - {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, -] - -[package.dependencies] -ptyprocess = ">=0.5" - -[[package]] -name = "platformdirs" -version = "4.3.8" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -optional = false -python-versions = ">=3.9" -files = [ - {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, - {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, -] - -[package.extras] -docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.4)", "pytest-cov (>=6)", "pytest-mock (>=3.14)"] -type = ["mypy (>=1.14.1)"] - -[[package]] -name = "pre-commit" -version = "3.8.0" -description = "A framework for managing and maintaining multi-language pre-commit hooks." -optional = false -python-versions = ">=3.9" -files = [ - {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, - {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, -] - -[package.dependencies] -cfgv = ">=2.0.0" -identify = ">=1.0.0" -nodeenv = ">=0.11.1" -pyyaml = ">=5.1" -virtualenv = ">=20.10.0" - -[[package]] -name = "prompt-toolkit" -version = "3.0.51" -description = "Library for building powerful interactive command lines in Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "prompt_toolkit-3.0.51-py3-none-any.whl", hash = "sha256:52742911fde84e2d423e2f9a4cf1de7d7ac4e51958f648d9540e0fb8db077b07"}, - {file = "prompt_toolkit-3.0.51.tar.gz", hash = "sha256:931a162e3b27fc90c86f1b48bb1fb2c528c2761475e57c9c06de13311c7b54ed"}, -] - -[package.dependencies] -wcwidth = "*" - -[[package]] -name = "ptyprocess" -version = "0.7.0" -description = "Run a subprocess in a pseudo terminal" -optional = false -python-versions = "*" -files = [ - {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, - {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, -] - -[[package]] -name = "pure-eval" -version = "0.2.3" -description = "Safely evaluate AST nodes without side effects" -optional = false -python-versions = "*" -files = [ - {file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"}, - {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "pygments" -version = "2.19.2" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, - {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "pyyaml" -version = "6.0.2" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, - {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, - {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, - {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, - {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, - {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, - {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, - {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, - {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, - {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, - {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, - {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, - {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, - {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, - {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, - {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, - {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, - {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, - {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, - {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, - {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, - {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, - {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, - {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, - {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, - {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, - {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, - {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, - {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, - {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, - {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, - {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, - {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, - {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, - {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, - {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, - {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, - {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, - {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, - {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, - {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, -] - -[[package]] -name = "stack-data" -version = "0.6.3" -description = "Extract data from python stack frames and tracebacks for informative displays" -optional = false -python-versions = "*" -files = [ - {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, - {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, -] - -[package.dependencies] -asttokens = ">=2.1.0" -executing = ">=1.2.0" -pure-eval = "*" - -[package.extras] -tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] - -[[package]] -name = "tomli" -version = "2.2.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.8" -files = [ - {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, - {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, - {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, - {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, - {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, - {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, - {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, - {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, - {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, - {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, - {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, - {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, - {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, - {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, - {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, - {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, - {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, - {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, - {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, - {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, - {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, - {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, - {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, - {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, - {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, - {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, - {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, - {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, - {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, - {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, - {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, - {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, -] - -[[package]] -name = "traitlets" -version = "5.14.3" -description = "Traitlets Python configuration system" -optional = false -python-versions = ">=3.8" -files = [ - {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"}, - {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"}, -] - -[package.extras] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] -test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"] - -[[package]] -name = "typing-extensions" -version = "4.14.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -files = [ - {file = "typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af"}, - {file = "typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4"}, -] - -[[package]] -name = "virtualenv" -version = "20.31.2" -description = "Virtual Python Environment builder" -optional = false -python-versions = ">=3.8" -files = [ - {file = "virtualenv-20.31.2-py3-none-any.whl", hash = "sha256:36efd0d9650ee985f0cad72065001e66d49a6f24eb44d98980f630686243cf11"}, - {file = "virtualenv-20.31.2.tar.gz", hash = "sha256:e10c0a9d02835e592521be48b332b6caee6887f332c111aa79a09b9e79efc2af"}, -] - -[package.dependencies] -distlib = ">=0.3.7,<1" -filelock = ">=3.12.2,<4" -platformdirs = ">=3.9.1,<5" - -[package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] - -[[package]] -name = "wcwidth" -version = "0.2.13" -description = "Measures the displayed width of unicode strings in a terminal" -optional = false -python-versions = "*" -files = [ - {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, - {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, -] - -[metadata] -lock-version = "2.0" -python-versions = ">=3.9,<3.13" -content-hash = "b06d8f3227a29a324890821df9d59fb02a8ef9b93f3db40e2968f95b402f767f" diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 8049116c7..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -[tool.poetry] -name = "truss-examples" -version = "0.1.0" -description = "" -authors = ["Truss Maintainers "] -license = "MIT" -readme = "README.md" -package-mode = false - -[tool.poetry.dependencies] -python = ">=3.9,<3.13" - -[tool.poetry.group.dev.dependencies] -black = "^23.7.0" -ipython = "^8.14.0" -isort = "^5.12.0" -pre-commit = "^3.5.0" - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" diff --git a/qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/README.md b/qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/README.md deleted file mode 100644 index c22cf3181..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-0.6B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-0.6B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-0.6B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" -) -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-0.6B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/config.yaml b/qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/config.yaml deleted file mode 100644 index 1dd6b0171..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-0.6b-A10G/config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# this file was autogenerated by `generate_templates.py` - please do change via template only -model_metadata: - example_model_input: - encoding_format: float - input: - [ - "Baseten Embeddings are high throughput.", - "Baseten Embeddings are low latency.", - ] - model: model - tags: - - openai-compatible -model_name: Qwen3 Embedding 0.6B -python_version: py39 -resources: - accelerator: A10G - cpu: "1" - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-0.6B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - runtime: - webserver_default_route: /v1/embeddings diff --git a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/README.md b/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/README.md deleted file mode 100644 index c22cf3181..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-0.6B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-0.6B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-0.6B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" -) -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-0.6B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/config.yaml b/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/config.yaml deleted file mode 100644 index 6469b634a..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8-h100/config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# this file was autogenerated by `generate_templates.py` - please do change via template only -model_metadata: - example_model_input: - encoding_format: float - input: - [ - "Baseten Embeddings are high throughput.", - "Baseten Embeddings are low latency.", - ] - model: model - tags: - - openai-compatible -model_name: Qwen3 Embedding 0.6B -python_version: py39 -resources: - accelerator: H100_40GB - cpu: "1" - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-0.6B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings diff --git a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md b/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md deleted file mode 100644 index c22cf3181..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-0.6B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-0.6B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-0.6B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-0.6B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-0.6b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" -) -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-0.6b-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-0.6B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/config.yaml b/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/config.yaml deleted file mode 100644 index 4461d6505..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-0.6b-fp8/config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# this file was autogenerated by `generate_templates.py` - please do change via template only -model_metadata: - example_model_input: - encoding_format: float - input: ["Baseten Embeddings are high throughput.", "Baseten Embeddings are low latency."] - model: model - tags: - - openai-compatible -model_name: Qwen3 Embedding 0.6B -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-0.6B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 diff --git a/qwen/BEI-qwen-qwen3-embedding-4b-fp8/README.md b/qwen/BEI-qwen-qwen3-embedding-4b-fp8/README.md deleted file mode 100644 index 91dd4fa44..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-4b-fp8/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-4B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-4B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-Embedding-4B](https://huggingface.co/Qwen/Qwen3-Embedding-4B). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Qwen/Qwen3-Embedding-4B is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-4b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-4b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" -) -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-embedding-4b-fp8/config.yaml b/qwen/BEI-qwen-qwen3-embedding-4b-fp8/config.yaml deleted file mode 100644 index e2e56e9ba..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-4b-fp8/config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# this file was autogenerated by `generate_templates.py` - please do change via template only -model_metadata: - example_model_input: - encoding_format: float - input: ["Instruct: {task_description}\nQuery:{query}", "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:What is the capital of China?"] - model: model -model_name: library-model-BEI-qwen3-embedding-4b -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-4B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - version_overrides: - bei_version: 0.0.23 - engine_builder_version: 0.18.1.post10.dev1 diff --git a/qwen/BEI-qwen-qwen3-embedding-8b-fp8/README.md b/qwen/BEI-qwen-qwen3-embedding-8b-fp8/README.md deleted file mode 100644 index 4c13b4c88..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-8b-fp8/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-8B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Embedding-8B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Embedding-8B-auto](https://huggingface.co/michaelfeil/Qwen3-Embedding-8B-auto). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -michaelfeil/Qwen3-Embedding-8B-auto is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-embedding-8b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-embedding-8b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" -) -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - encoding_format: float - input: text string - model: model -model_name: BEI-qwen-qwen3-embedding-8b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-8B-auto - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-embedding-8b-fp8/config.yaml b/qwen/BEI-qwen-qwen3-embedding-8b-fp8/config.yaml deleted file mode 100644 index 4fcadd06f..000000000 --- a/qwen/BEI-qwen-qwen3-embedding-8b-fp8/config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -model_metadata: - example_model_input: - encoding_format: float - input: ["Baseten Embeddings are high throughput.", "Baseten Embeddings are low latency."] - model: model - tags: - - openai-compatible -model_name: Qwen3 Embedding 8B -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Embedding-8B-auto - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /v1/embeddings - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 diff --git a/qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md b/qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md deleted file mode 100644 index 89cf966fe..000000000 --- a/qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/README.md +++ /dev/null @@ -1,142 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-0.6B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-0.6B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Reranker-0.6B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-0.6B-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/Qwen3-Reranker-0.6B-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-0.6b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-reranker-0.6b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - Baseten is a fast inference provider - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-qwen-qwen3-reranker-0.6b-fp8-truss-example -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-0.6B-seq - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 4 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/config.yaml b/qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/config.yaml deleted file mode 100644 index 7b5e9c718..000000000 --- a/qwen/BEI-qwen-qwen3-reranker-0.6b-fp8/config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -model_metadata: - example_model_input: - inputs: - # advanced formatting of the string needed: - - "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n: {intruction}\n: {query}\n: {doc}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" - - "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n: Given a web search query, retrieve relevant passages that answer the query\n: What is the capital of China?\n: The capital of China is Beijing.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" - raw_scores: true - truncate: true - truncation_direction: Right -model_name: Qwen3 Reranker 0.6B -python_version: py39 -resources: - accelerator: L4 - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-0.6B-seq - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 diff --git a/qwen/BEI-qwen-qwen3-reranker-4b-fp8/README.md b/qwen/BEI-qwen-qwen3-reranker-4b-fp8/README.md deleted file mode 100644 index c77ddb5f7..000000000 --- a/qwen/BEI-qwen-qwen3-reranker-4b-fp8/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-4B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-4B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [Qwen/Qwen3-Reranker-4B](https://huggingface.co/Qwen/Qwen3-Reranker-4B). -Suitable models need to have the configurations of the `sentence-transformers` library, which are used for embeddings. Such repos contain e.g. a `sbert_config.json` or a `1_Pooling/config.json` file besides the fast-tokenizer and the safetensors file. - -Qwen/Qwen3-Reranker-4B is a text-embeddings model, producing a 1D embeddings vector, given an input. -It's frequently used for downstream tasks like clustering, used with vector databases. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-4b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-reranker-4b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings` -```json -{ - "encoding_format": "float", # or base64 - "input": "string", # can be list of strings for multiple embeddings - "model": "null", - "user": "null" -} -``` - -Returns: -```json -{ - "data": [ - { - "embedding": [ - 0 - ], - "index": 0, - "object": "embedding" - } - ], - "model": "thenlper/gte-base", - "object": "list", - "usage": { - "prompt_tokens": 512, - "total_tokens": 512 - } -} -``` -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/v1/embeddings` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### curl -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string", "model": "model"}' -``` - -### OpenAI compatible client library -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ['BASETEN_API_KEY'], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -embedding = client.embeddings.create( - input="Baseten Embeddings are fast", - model="model" -) -``` -### requests python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/embeddings", - headers={"Authorization": "Api-Key " + str(os.environ['BASETEN_API_KEY'])}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-reranker-4b-fp8/config.yaml b/qwen/BEI-qwen-qwen3-reranker-4b-fp8/config.yaml deleted file mode 100644 index da6222016..000000000 --- a/qwen/BEI-qwen-qwen3-reranker-4b-fp8/config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -model_metadata: - example_model_input: - inputs: - # advanced formatting of the string needed: - - - "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n: {intruction}\n: {query}\n: {doc}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" - - - "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n: Given a web search query, retrieve relevant passages that answer the query\n: What is the capital of China?\n: The capital of China is Beijing.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI Qwen3 Reranker 4b fp8 -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-4B-seq - revision: main - source: HF - max_num_tokens: 32768 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - version_overrides: - bei_version: 0.0.23 - engine_builder_version: 0.18.1.post10.dev1 diff --git a/qwen/BEI-qwen-qwen3-reranker-8b-fp8/README.md b/qwen/BEI-qwen-qwen3-reranker-8b-fp8/README.md deleted file mode 100644 index 701a13667..000000000 --- a/qwen/BEI-qwen-qwen3-reranker-8b-fp8/README.md +++ /dev/null @@ -1,142 +0,0 @@ -# BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-8B - -This is a Deployment for BEI (Baseten-Embeddings-Inference) with Qwen/Qwen3-Reranker-8B. BEI is Baseten's solution for production-grade deployments via TensorRT-LLM for (text) embeddings, reranking models and prediction models. -With BEI you get the following benefits: -- *Lowest-latency inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama)1 -- *Highest-throughput inference* across any embedding solution (vLLM, SGlang, Infinity, TEI, Ollama) - thanks to XQA kernels, FP8 and dynamic batching.2 -- High parallelism: up to 1400 client embeddings per second -- Cached model weights for fast vertical scaling and high availability - no Hugging Face hub dependency at runtime - - -# Examples: -This deployment is specifically designed for the Hugging Face model [michaelfeil/Qwen3-Reranker-8B-seq](https://huggingface.co/michaelfeil/Qwen3-Reranker-8B-seq). -Suitable models can be identified by the `ForSequenceClassification` suffix in the model name. Prediction models may have one or more labels, which are returned with the prediction. - -michaelfeil/Qwen3-Reranker-8B-seq is a text-classification model, used to classify a text into a category. \nIt is frequently used in sentiment analysis, spam detection, and more. It's also used for deployment of chat rating models, e.g. RLHF reward models or toxicity detection models. - -This model is quantized to FP8 for deployment, which is supported by Nvidia's newest GPUs e.g. H100, H100_40GB or L4. Quantization is optional, but leads to higher efficiency. - -## Deployment with Truss - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - - -First, clone this repository: -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd 11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8 -``` - -With `11-embeddings-reranker-classification-tensorrt/BEI-qwen-qwen3-reranker-8b-fp8` as your working directory, you can deploy the model with the following command. Paste your Baseten API key if prompted. - -```sh -truss push --publish -# prints: -# ✨ Model BEI-qwen-qwen3-reranker-8b-fp8-truss-example was successfully pushed ✨ -# πŸͺ΅ View logs for your deployment at https://app.baseten.co/models/yyyyyy/logs/xxxxxx -``` - -## Call your model - -### API-Schema: -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/sync/predict` -```json -{ - "inputs": "Baseten is a fast inference provider", - "raw_scores": true, - "truncate": true, - "truncation_direction": "Right" -} -``` - -```python -import requests -import os - -headers = { - f"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}" -} - -requests.post( - headers=headers, - url="https://model-xxxxxx.api.baseten.co/environments/production/sync/predict", - json={ - "inputs": [["Baseten is a fast inference provider", ["classify this separately."]], - "raw_scores": True, - "truncate": True, - "truncation_direction": "Right" - } -) -``` -Returns: -```json -[ - [ - { - "label": "excitement", - "score": 0.99 - } - ], - [ - { - "label": "excitement", - "score": 0.01 - } - ] -] -``` -Important, this is different from the `predict` route that you usually call. (https://model-xxxxxx.api.baseten.co/environments/production/predict), it contains an additional `sync` before that. -The OpenAPI.json is available under https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json for more details. - -#### Advanced: -You may also use Baseten's async jobs API, which returns a request_id, which you can use to query the status of the job and get the results. - -POST-Route: `https://model-xxxxxx.api.baseten.co/environments/production/async/predict` -Read more about [Baseten's Async API here](https://docs.baseten.co/invoke/async) - -### OpenAI compatible client library -OpenAI does not have a classification endpoint, therefore no client library is available. - - -## Config.yaml -By default, the following configuration is used for this deployment. This config uses `quantization_type=fp8`. This is optional, remove the `quantization_type` field or set it to `no_quant` for float16/bfloat16. - -```yaml -model_metadata: - example_model_input: - inputs: - - Baseten is a fast inference provider - - Classify this separately. - raw_scores: true - truncate: true - truncation_direction: Right -model_name: BEI-qwen-qwen3-reranker-8b-fp8-truss-example -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-8B-seq - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 - -``` - -## Support -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/qwen/BEI-qwen-qwen3-reranker-8b-fp8/config.yaml b/qwen/BEI-qwen-qwen3-reranker-8b-fp8/config.yaml deleted file mode 100644 index b00012f5c..000000000 --- a/qwen/BEI-qwen-qwen3-reranker-8b-fp8/config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -model_metadata: - example_model_input: - inputs: - # advanced formatting of the string needed: - - "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n: {intruction}\n: {query}\n: {doc}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" - - "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n: Given a web search query, retrieve relevant passages that answer the query\n: What is the capital of China?\n: The capital of China is Beijing.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" - raw_scores: true - truncate: true - truncation_direction: Right -model_name: Qwen3 Reranker 8B -python_version: py39 -resources: - accelerator: H100_40GB - cpu: '1' - memory: 10Gi - use_gpu: true -trt_llm: - build: - base_model: encoder - checkpoint_repository: - repo: michaelfeil/Qwen3-Reranker-8B-seq - revision: main - source: HF - max_num_tokens: 40960 - num_builder_gpus: 1 - quantization_type: fp8 - runtime: - webserver_default_route: /predict - version_overrides: - bei_version: 0.0.25-b200-dev-v4 - engine_builder_version: 0.20.0.dev1 diff --git a/qwen/engine-qwen-2-5-14b-coder-instruct/README.md b/qwen/engine-qwen-2-5-14b-coder-instruct/README.md deleted file mode 100644 index dbea81234..000000000 --- a/qwen/engine-qwen-2-5-14b-coder-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen Coder 2.5 14B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen Coder 2.5 14B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-14b-instruct/README.md b/qwen/engine-qwen-2-5-14b-instruct/README.md deleted file mode 100644 index b83ed1d76..000000000 --- a/qwen/engine-qwen-2-5-14b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen 2.5 14B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen 2.5 14B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-32b-coder-instruct/README.md b/qwen/engine-qwen-2-5-32b-coder-instruct/README.md deleted file mode 100644 index 962660240..000000000 --- a/qwen/engine-qwen-2-5-32b-coder-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen Coder 2.5 32B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen Coder 2.5 32B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-32b-instruct/README.md b/qwen/engine-qwen-2-5-32b-instruct/README.md deleted file mode 100644 index bb04b18d1..000000000 --- a/qwen/engine-qwen-2-5-32b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen 2.5 32B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen 2.5 32B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-3b-instruct/README.md b/qwen/engine-qwen-2-5-3b-instruct/README.md deleted file mode 100644 index 30b10008e..000000000 --- a/qwen/engine-qwen-2-5-3b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen 2.5 3B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen 2.5 3B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-72b-instruct/README.md b/qwen/engine-qwen-2-5-72b-instruct/README.md deleted file mode 100644 index 9d5733ae4..000000000 --- a/qwen/engine-qwen-2-5-72b-instruct/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Qwen 2.5 72B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen 2.5 72B Instruct. - -Note that while other sizes of Qwen 2.5 are licensed as Apache 2.0, 72B sizes use the [qwen license](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct/blob/main/LICENSE). - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-72b-math-instruct/README.md b/qwen/engine-qwen-2-5-72b-math-instruct/README.md deleted file mode 100644 index ce83e62b5..000000000 --- a/qwen/engine-qwen-2-5-72b-math-instruct/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Qwen Math 2.5 72B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen Math 2.5 72B Instruct. - -Note that while other sizes of Qwen 2.5 are licensed as Apache 2.0, 72B sizes use the [qwen license](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct/blob/main/LICENSE). - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-7b-coder-instruct/README.md b/qwen/engine-qwen-2-5-7b-coder-instruct/README.md deleted file mode 100644 index cac820a03..000000000 --- a/qwen/engine-qwen-2-5-7b-coder-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen Coder 2.5 7B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen Coder 2.5 7B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-7b-instruct/README.md b/qwen/engine-qwen-2-5-7b-instruct/README.md deleted file mode 100644 index 34d234473..000000000 --- a/qwen/engine-qwen-2-5-7b-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen 2.5 7B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen 2.5 7B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/engine-qwen-2-5-7b-math-instruct/README.md b/qwen/engine-qwen-2-5-7b-math-instruct/README.md deleted file mode 100644 index 4ca182e64..000000000 --- a/qwen/engine-qwen-2-5-7b-math-instruct/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Qwen Math 2.5 7B Instruct Engine - -This example uses the [TensorRT-LLM Engine Builder for Qwen](https://docs.baseten.co/performance/examples/qwen-trt) to build and deploy an optimized inference engine for Qwen Math 2.5 7B Instruct. - -For advanced control over the engine building process, see [engine control in Python](https://docs.baseten.co/performance/engine-builder-customization) and [engine builder configuration](https://docs.baseten.co/performance/engine-builder-config) docs. diff --git a/qwen/qwen-3-30B-A3-coder/README.md b/qwen/qwen-3-30B-A3-coder/README.md deleted file mode 100644 index f502daf2d..000000000 --- a/qwen/qwen-3-30B-A3-coder/README.md +++ /dev/null @@ -1,162 +0,0 @@ -# Qwen3-Coder-30B-A3B-Instruct Model - -This Truss serves the Qwen3-Coder-30B-A3B-Instruct model, a powerful coding-focused language model that excels at agentic coding tasks. The model is based on the [Qwen/Qwen3-Coder-30B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct) model from Hugging Face and is optimized for high-performance coding assistance. It is Apache 2.0 licensed and can be used commercially without restrictions. - -## Model Description - -The Qwen3-Coder-30B-A3B-Instruct model is a specialized coding language model that features: - -- **Agentic Coding**: Excellent performance on agentic coding tasks and browser-use scenarios -- **Long Context**: Native support for 256K tokens, extendable up to 1M tokens with Yarn -- **Function Calling**: Specialized function call format for tool integration -- **Repository-Scale Understanding**: Optimized for understanding large codebases -- **Streaming Support**: Real-time code generation with streaming capabilities - -## Model Parameters - -The model accepts the following parameters: - -- `messages` (required): Array of message objects with role and content -- `model` (optional): Model name (default: "Qwen/Qwen3-Coder-30B-A3B-Instruct") -- `max_tokens` (optional): Maximum tokens to generate (default: 1024) -- `temperature` (optional): Sampling temperature (default: 0.7) -- `stream` (optional): Enable streaming response (default: true) -- `tools` (optional): Array of function definitions for tool calling - -## Example Usage - -The model outputs structured responses compatible with OpenAI's chat completion format. - -```python -import httpx -import os - -# Replace with your model ID and API key -model_id = "your-model-id" -baseten_api_key = os.environ["BASETEN_API_KEY"] - -# Example 1: Basic code generation -basic_data = { - "messages": [ - {"role": "system", "content": "You are a helpful coding assistant."}, - {"role": "user", "content": "Write a quick sort algorithm in Python."} - ], - "max_tokens": 1024, - "temperature": 0.7, - "stream": True -} - -# Example 2: Function calling for tool integration -def square_the_number(num: float) -> dict: - return {"result": num ** 2} - -tools_data = { - "messages": [ - {"role": "user", "content": "Calculate the square of 1024"} - ], - "tools": [ - { - "type": "function", - "function": { - "name": "square_the_number", - "description": "Calculate the square of a number", - "parameters": { - "type": "object", - "required": ["num"], - "properties": { - "num": { - "type": "number", - "description": "The number to square" - } - } - } - } - } - ], - "max_tokens": 1024, - "temperature": 0.7 -} - -# Call the model -print("Generating code...") -response = httpx.post( - f"https://model-{model_id}.api.baseten.co/development/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=basic_data, - timeout=httpx.Timeout(60.0) -) - -# Get the result -result = response.json() -print("Generated code:", result.get("choices", [{}])[0].get("message", {}).get("content", "")) -``` - -## Agentic Coding Examples - -The model excels at agentic coding tasks. Here are some example use cases: - -```python -# Repository analysis -repo_analysis = { - "messages": [ - {"role": "user", "content": "Analyze this codebase and suggest improvements for the authentication system."} - ], - "max_tokens": 2048 -} - -# Code review -code_review = { - "messages": [ - {"role": "user", "content": "Review this Python function for security vulnerabilities:\n\ndef process_user_input(data):\n return eval(data)"} - ], - "max_tokens": 1024 -} - -# Debugging assistance -debugging = { - "messages": [ - {"role": "user", "content": "Help me debug this error: 'TypeError: 'NoneType' object is not callable'"} - ], - "max_tokens": 1024 -} -``` - -## Best Practices - -For optimal performance, we recommend: - -1. **Sampling Parameters**: - - Temperature: 0.7 - - Top-p: 0.8 - - Top-k: 20 - - Repetition penalty: 1.05 - -2. **Context Length**: Use up to 65,536 tokens for most queries - -3. **Streaming**: Enable streaming for real-time code generation - -4. **Function Calling**: Define clear tool schemas for agentic tasks - -## Deployment - -To deploy this model: - -1. Clone the repository -2. Make sure you have the Truss CLI installed (`pip install truss`) -3. Run the deployment command: - -```bash -truss push qwen/qwen-3-30B-A3-coder --publish -``` - -## Model Features - -- **OpenAI-Compatible API**: Full compatibility with OpenAI's chat completion format -- **Streaming Support**: Real-time response streaming for better user experience -- **Tool Calling**: Native support for function calling and tool integration -- **Long Context**: Handles large codebases and documentation -- **GPU Optimization**: Optimized for H100 GPUs with SGLang - -## License - -This model is licensed under Apache 2.0 and can be used commercially without restrictions. diff --git a/qwen/qwen-3-30b-omni-thinker/README.md b/qwen/qwen-3-30b-omni-thinker/README.md deleted file mode 100644 index 18f0eacf0..000000000 --- a/qwen/qwen-3-30b-omni-thinker/README.md +++ /dev/null @@ -1,256 +0,0 @@ -# Qwen3-Omni-30B-Thinker (vLLM) on Baseten - -This is a [Truss](https://truss.baseten.co/) for serving the "thinker" part of **Qwen/Qwen3-Omni-30B-A3B-Intruct** with **vLLM** on Baseten. It exposes an **OpenAI-compatible /v1/chat/completions** endpoint that accepts **text, image, audio, and video** inputs and returns low-latency text responses. - -**Why this deployment** - -* *Multimodal, end-to-end*: text, images, audio, and video inputs in a single chat request; text responses today (audio TTS output is not compatible with vLLM at the moment). -* *OpenAI-compatible API*: drop-in with `openai` libraries and many tools. -* *Production-ready on Baseten*: autoscaling, logs, metrics, and zero-downtime publishes. - ---- - -# Overview - -**Model**: `Qwen/Qwen3-Omni-30B-A3B-Instruct` -**Modalities**: - -* **Input**: text, image(s), audio, video -* **Output**: text - ---- - -# Deploy with Truss - -Before deployment: - -1. Create a [Baseten account](https://app.baseten.co/signup) and an [API key](https://app.baseten.co/settings/account/api_keys). -2. Install Truss: `pip install --upgrade truss` - -Clone the examples repo (or your project) and cd into your working directory: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd qwen/qwen-3-30b-omni-thinker -``` - -Publish: - -```sh -truss push --publish -# ✨ Model Qwen3 Omni 30B Instruct (Thinker Only) was successfully pushed ✨ -``` - ---- - -# Call your model - -Your deployment is OpenAI-compatible. Replace `model-xxxxxx` and include your Baseten API key. - -## API schema (Chat Completions) - -**POST** `https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/chat/completions` - -**Request (multimodal example)**: - -```json -{ - "model": "qwen3-omni", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": [ - {"type": "text", "text": "Describe what you see and hear."}, - { - "type": "image_url", - "image_url": {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"} - }, - { - "type": "audio_url", - "audio_url": {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cough.wav"} - } - ] - } - ], - "max_tokens": 2048, - "temperature": 0.7, - "stream": false -} -``` - -**Response (truncated example)**: - -```json -{ - "id": "chatcmpl-...", - "object": "chat.completion", - "created": 1710000000, - "model": "qwen3-omni", - "choices": [ - { - "index": 0, - "finish_reason": "stop", - "message": { - "role": "assistant", - "content": "I see several parked cars in front of a building and hear a short cough." - } - } - ], - "usage": { - "prompt_tokens": 512, - "completion_tokens": 24, - "total_tokens": 536 - } -} -``` - -### curl - -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/chat/completions \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "model":"qwen3-omni", - "messages":[ - {"role":"system","content":"You are a helpful assistant."}, - {"role":"user","content":[ - {"type":"text","text":"Describe this image and audio content."}, - {"type":"image_url","image_url":{"url":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"}}, - {"type":"audio_url","audio_url":{"url":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cough.wav"}} - ]} - ], - "max_tokens":2048, - "temperature":0.7, - "stream":false - }' -``` - -### OpenAI Python SDK - -```python -from openai import OpenAI -import os - -client = OpenAI( - api_key=os.environ["BASETEN_API_KEY"], - base_url="https://model-xxxxxx.api.baseten.co/environments/production/sync/v1" -) - -resp = client.chat.completions.create( - model="qwen3-omni", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": [ - {"type": "text", "text": "Describe this image and audio content."}, - {"type": "image_url", "image_url": {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"}}, - {"type": "audio_url", "audio_url": {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cough.wav"}} - ]} - ], - max_tokens=2048, - temperature=0.7, - stream=False, -) -print(resp.choices[0].message.content) -``` - -### Streaming (server-sent events) - -```python -stream = client.chat.completions.create( - model="qwen3-omni", - messages=[{"role":"user","content":"Summarize this short clip."}], - stream=True -) -for event in stream: - if event.choices and event.choices[0].delta: - print(event.choices[0].delta.content or "", end="", flush=True) -``` - -### requests (Python) - -```python -import os, requests, json - -url = "https://model-xxxxxx.api.baseten.co/environments/production/sync/v1/chat/completions" -headers = { - "Authorization": "Api-Key " + os.environ["BASETEN_API_KEY"], - "Content-Type": "application/json" -} -payload = { - "model": "qwen3-omni", - "messages": [{"role":"user","content":[{"type":"text","text":"Briefly describe the image."}, - {"type":"image_url","image_url":{"url":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"}}]}], - "max_tokens": 512 -} -print(requests.post(url, headers=headers, data=json.dumps(payload)).json()) -``` - -> The OpenAPI schema is available at: -> `https://model-xxxxxx.api.baseten.co/environments/production/sync/openapi.json` - ---- - -# Advanced usage notes - -* **Images, audio, video**: Provide remote URLs via `image_url` / `audio_url` / `video_url`, or configure `--allowed-local-media-path` to allow local file ingestion. -* **Batching/throughput**: Configure Baseten `predict_concurrency` for request-level concurrency. vLLM will also batch internally. -* **Long contexts**: `--max-model-len 65,536` in the server command; adjust based on memory. -* **Multi-GPU**: Add `-tp N` (tensor parallelism) in the vLLM command to shard across GPUs. -* **Audio output**: If you require generated speech, verify your vLLM build and model variant support it; otherwise pipe text into a TTS stage. - ---- - -**Notes** - -* The container image `qwenllm/qwen3-omni:3-cu124` bundles vLLM and dependencies. -* For **multi-GPU** boxes, add `-tp ` to the `vllm serve` command. - ---- - -# Examples - -### 1) Pure text - -```json -{ - "model":"qwen3-omni", - "messages":[{"role":"user","content":"Give me three creative app ideas for teachers."}], - "max_tokens":512 -} -``` - -### 2) Image + instruction - -```json -{ - "model":"qwen3-omni", - "messages":[ - {"role":"user","content":[ - {"type":"image_url","image_url":{"url":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"}}, - {"type":"text","text":"Describe the scene in one sentence."} - ]} - ] -} -``` - -### 3) Audio + question - -```json -{ - "model":"qwen3-omni", - "messages":[ - {"role":"user","content":[ - {"type":"audio_url","audio_url":{"url":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cough.wav"}}, - {"type":"text","text":"What do you hear?"} - ]} - ] -} -``` - ---- - -# Support - -If you have questions or need help, open an issue in this repository or contact Baseten support. diff --git a/qwen/qwen-3-30b-omni/README.md b/qwen/qwen-3-30b-omni/README.md deleted file mode 100644 index 5d3726dfd..000000000 --- a/qwen/qwen-3-30b-omni/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# Qwen3-Omni-30B-Thinker (vLLM) on Baseten - -This is a [Truss](https://truss.baseten.co/) for serving **Qwen/Qwen3-Omni-30B-A3B-Intruct** with **transformers** on Baseten. It exposes an endpoint that accepts **text, image, audio, and video** inputs and returns low-latency text and audio responses. - -**Why this deployment** - -* *Multimodal, end-to-end*: text, images, audio, and video inputs in a single chat request; text and spoken audio responses. -* *Production-ready on Baseten*: autoscaling, logs, metrics, and zero-downtime publishes. - ---- - -# Overview - -**Model**: `Qwen/Qwen3-Omni-30B-A3B-Instruct` -**Modalities**: - -* **Input**: text, image(s), audio, video -* **Output**: text, audio - ---- - -# Deploy with Truss - -Before deployment: - -1. Create a [Baseten account](https://app.baseten.co/signup) and an [API key](https://app.baseten.co/settings/account/api_keys). -2. Install Truss: `pip install --upgrade truss` - -Clone the examples repo (or your project) and cd into your working directory: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd qwen/qwen-3-30b-omni-thinker -``` - -Publish: - -```sh -truss push --publish -# ✨ Model Qwen3 Omni 30B Instruct was successfully pushed ✨ -``` - ---- - -# Call your model - -Replace `model-xxxxxx` and include your Baseten API key. - -## API schema (Chat Completions) - -**POST** `https://model-xxxxxx.api.baseten.co/development/predict` - -**Request example**: - -```json -{ - "speaker": "Chelsie", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello how are you?"} - ] - } - ] -} -``` - -**Response (truncated example)**: - -```json -{ - "text": "Hello! I'm doing well, thank you. How can I assist you today?", - "audio": "UklGRs5hAwBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAAZGF0YaphAwAE..." -} -``` - -### curl - -```bash -curl -X POST https://model-xxxxxx.api.baseten.co/development/predict \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "speaker":"Aiden", - "messages":[ - {"role":"system","content":"You are a helpful assistant."}, - {"role":"user","content":[ - {"type":"text","text":"Describe this image and audio content."}, - {"type":"image","image":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"}, - {"type":"audio","audio":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cough.wav"} - ]} - ] - }' -``` - -### requests (Python) - -```python -import os, requests, json, base64 - -url = "https://model-xxxxxx.api.baseten.co/development/predict" -headers = { - "Authorization": "Api-Key " + os.environ["BASETEN_API_KEY"], - "Content-Type": "application/json" -} -payload = { - "speaker": "Ethan", - "messages": [{"role":"user","content":[{"type":"text","text":"Briefly describe the image."}, - {"type":"image","image":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"}]}] -} -result = requests.post(url, headers=headers, data=json.dumps(payload)).json() - -print(result["text"]) - -with open("result.wav", "wb") as f: - f.write(base64.b64decode(result["audio"])) -``` - ---- - -**Notes** - -* The container image `qwenllm/qwen3-omni:3-cu124` bundles flash attention and dependencies. - ---- - -# Examples - -### 1) Pure text - -```json -{ - "model":"qwen3-omni", - "messages":[{"role":"user","content":"Give me three creative app ideas for teachers."}], - "max_tokens":512 -} -``` - -### 2) Image + instruction - -```json -{ - "model":"qwen3-omni", - "messages":[ - {"role":"user","content":[ - {"type":"image","image":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cars.jpg"}, - {"type":"text","text":"Describe the scene in one sentence."} - ]} - ] -} -``` - -### 3) Audio + question - -```json -{ - "model":"qwen3-omni", - "messages":[ - {"role":"user","content":[ - {"type":"audio","audio":"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/cough.wav"}, - {"type":"text","text":"What do you hear?"} - ]} - ] -} -``` - ---- - -# Support - -If you have questions or need help, open an issue in this repository or contact Baseten support. diff --git a/qwen/qwen-3-asr/README.md b/qwen/qwen-3-asr/README.md deleted file mode 100644 index af3a581d5..000000000 --- a/qwen/qwen-3-asr/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Qwen3 ASR 1.7B - -This example shows how to call a Baseten deployment using the OpenAI Python SDK to run **Qwen/Qwen3-ASR-1.7B** on an audio URL. - -## Prerequisites - -- Python 3.9+ -- OpenAI Python SDK installed: - -```bash -pip install openai -``` - -## Example: Transcribe an audio URL - -```python -from openai import OpenAI - -model_id = "" # place model ID here - -client = OpenAI( - api_key="BASETEN-API-KEY", - base_url=f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" -) - -response = client.chat.completions.create( - model="Qwen/Qwen3-ASR-1.7B", - stream=False, - messages=[ - { - "role": "user", - "content": [ - { - "type": "audio_url", - "audio_url": - {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav"} - - } - ] - } - ], -) - -print(response.choices[0].message.content) -``` - -## Sample Output -```txt -Response: language EnglishUh huh. Oh yeah, yeah. He wasn't even that big when I started listening to him, but and his solo music didn't do overly well, but he did very well when he started writing for other people. - -``` diff --git a/qwen/qwen-image/README.md b/qwen/qwen-image/README.md deleted file mode 100644 index 2b6d62909..000000000 --- a/qwen/qwen-image/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# Qwen Image Model - -![Qwen Image on Baseten](assets/generated_image.jpg) - -This Truss serves the Qwen Image model, a powerful text-to-image generation model that supports both English and Chinese prompts. The model is based on the Qwen/Qwen-Image model from Hugging Face and is optimized for high-quality image generation. It is Apache 2.0 licensed and can be used commercially without restrictions. - -## Model Description - -The Qwen Image model is a diffusion-based text-to-image model that can generate high-quality images from text prompts. It features: - -- **Multilingual Support**: Handles both English and Chinese prompts -- **High Quality**: Generates 4K quality images with cinematic composition -- **Flexible Aspect Ratios**: Supports various image dimensions -- **Customizable Parameters**: Adjustable inference steps, guidance scale, and more - -## Model Parameters - -The model accepts the following parameters: - -- `prompt` (required): The text description of the image you want to generate -- `negative_prompt` (optional): Text describing what you don't want in the image (default: "") -- `width` (optional): Image width in pixels (default: 1024) -- `height` (optional): Image height in pixels (default: 1024) -- `num_inference_steps` (optional): Number of denoising steps (default: 50) -- `true_cfg_scale` (optional): Guidance scale for generation (default: 4.0) -- `seed` (optional): Random seed for reproducible results (default: random) - -## Example Usage - -The model outputs a base64 string which can be saved locally. - -```python -import httpx -import os -import base64 -from PIL import Image -from io import BytesIO - -# Replace with your model ID and API key -model_id = "your-model-id" -baseten_api_key = os.environ["BASETEN_API_KEY"] - -def b64_to_pil(b64_str): - """Convert base64 string to PIL image""" - return Image.open(BytesIO(base64.b64decode(b64_str))) - -# Example 1: English prompt -english_data = { - "prompt": "A fashionably dressed man on the streets of New York City holds a sign that says `Qwen Image on Baseten`", - "width": 1664, - "height": 928, - "num_inference_steps": 50, - "seed": 42 -} - -# Example 2: Chinese prompt -chinese_data = { - "prompt": "δΈ€εͺε―ηˆ±ηš„ε°ηŒ«εεœ¨θŠ±ε›­ι‡ŒοΌŒι˜³ε…‰ζ˜Žεͺš", - "width": 1024, - "height": 1024, - "num_inference_steps": 50 -} - -# Call the model with extended timeout for image generation -print("Generating image... This may take a moment.") -response = httpx.post( - f"https://model-{model_id}.api.baseten.co/development/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=english_data, - timeout=httpx.Timeout(120.0) -) - -# Get the result -result = response.json() -image_b64 = result.get("data") - -# Convert to image and save -image = b64_to_pil(image_b64) -image.save("generated_image.jpg") -print("Image generated successfully! Saved as 'generated_image.jpg'") -``` - -## Aspect Ratio Examples - -The model supports various aspect ratios. Here are some common configurations: - -```python -# Square (1:1) -{"width": 1024, "height": 1024} - -# Landscape (16:9) -{"width": 1664, "height": 928} - -# Portrait (9:16) -{"width": 928, "height": 1664} - -# Traditional (4:3) -{"width": 1472, "height": 1140} - -# Portrait Traditional (3:4) -{"width": 1140, "height": 1472} -``` - -## Deployment - -To deploy this model: - -1. Clone the repository -2. Make sure you have the Truss CLI installed (`pip install truss`) -3. Run the deployment command: - -```bash -truss push qwen/qwen-image --publish -``` - -## Model Features - -- **Automatic Language Detection**: The model automatically detects Chinese vs English prompts and applies appropriate quality enhancements -- **Quality Enhancement**: Automatically adds "Ultra HD, 4K, cinematic composition" for English prompts or "θΆ…ζΈ…οΌŒ4KοΌŒη”΅ε½±ηΊ§ζž„ε›Ύ" for Chinese prompts -- **GPU Optimization**: Uses bfloat16 precision on CUDA devices for optimal performance -- **Base64 Output**: Returns images as base64-encoded strings for easy API integration - -## Requirements - -- CUDA-compatible GPU (recommended for optimal performance) -- Python 3.8+ -- PyTorch -- Diffusers library -- Transformers library diff --git a/qwen/qwen-vl/README.md b/qwen/qwen-vl/README.md deleted file mode 100644 index f92c0e7db..000000000 --- a/qwen/qwen-vl/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Qwen VL Truss - -This is a [Truss](https://truss.baseten.co/) for [Qwen-VL](https://huggingface.co/Qwen/Qwen-VL) which is a visual language model. Qwen is a family of models developed by Alibaba Cloud. This LLM supports both English and Chinese. - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd qwen-vl -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `qwen-vl` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - - -## Qwen-VL API documentation - -This section provides an overview of the Qwen-VL model, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The instruction the model will follow to extract the data from the image. -- __image__ : The input image in the form of a URL or a base64 string. - - -## Example usage - -```python -import requests -import base64 -from PIL import Image -from io import BytesIO - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - - -data = { - "image": pil_to_b64(Image.open("/path/to/image/dog.jpg")), - "prompt": "Generate the caption in English with grounding" -} - -res = requests.post( - "https://model-.api.baseten.co/development/predict", - headers={"Authorization": "Api-Key "}, - json=data, -) - -print(res.json()) -``` - -## Example Output - -```json -{"output": "Picture 1: /tmp/tmpw6m_zmbk.png\nGenerate the caption in English with grounding A maltese dog(385,361),(783,934) in a flower garden<|endoftext|>"} -``` -![galu2](https://github.com/basetenlabs/truss-examples/assets/15642666/459d5a5a-37b0-49aa-830c-d933840c40a2) diff --git a/segment-anything/README.md b/segment-anything/README.md deleted file mode 100644 index 245d448db..000000000 --- a/segment-anything/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Segment Anything Model - -This is an example deploying Segment Anything Model (SAM) with truss weights preloaded - -## Deploy to Baseten -To deploy the model, run the following from the root of the directory - -``` -truss push --publish -``` - -## Predict -Example prediction: - -``` -truss predict --published -d '{"image_url": "https://as2.ftcdn.net/v2/jpg/00/66/26/87/1000_F_66268784_jccdcfdpf2vmq5X8raYA8JQT0sziZ1H9.jpg"}' -``` diff --git a/stable-diffusion/dreamshaper-lcm/README.md b/stable-diffusion/dreamshaper-lcm/README.md deleted file mode 100644 index d43c92122..000000000 --- a/stable-diffusion/dreamshaper-lcm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Dreamshaper Latent Consistency Model - -A Truss for [Dreamshaper LCM](https://huggingface.co/spaces/SimianLuo/Latent_Consistency_Model), a distillation of Dreamshaper (a Stable Diffusion 1.5 fine-tune), that can achieve similar quality in ~1-8 steps. Generate high quality 768 x 768 images in under a second. diff --git a/stable-diffusion/dreamshaper-lcm/config.yaml b/stable-diffusion/dreamshaper-lcm/config.yaml deleted file mode 100644 index fd170844b..000000000 --- a/stable-diffusion/dreamshaper-lcm/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_name: Dreamshaper Latent Consistency Model -python_version: py311 -requirements: -- diffusers=0.21.4 -- transformers=4.34.1 -- accelerate=0.23.0 -- torch=2.1.0 -resources: - accelerator: A10G - cpu: '1' - memory: 2Gi - use_gpu: true -secrets: {} -system_packages: [] diff --git a/stable-diffusion/playground-v2-trt/README.md b/stable-diffusion/playground-v2-trt/README.md deleted file mode 100644 index b447d5d6f..000000000 --- a/stable-diffusion/playground-v2-trt/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Playground v2 TensorRT Truss - -Playground is a diffusion-based text-to-image generative model. This README covers deploying and invoking this model. - -This model is packaged using [Truss](https://trussml.com), the simplest way to serve AI/ML models in production. - -## Deploy Playground v2 TensorRT - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion/playground-v2-trt -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `playground-v2-trt` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using SDXL through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -### Hardware notes - -Running inference on an A100 cuts invocation time to ~3.5 seconds. - -## Invoking Playground v2 TensorRT - -Playground v2 TensorRT returns an image in Base 64, which is not super useful as a string in your terminal. So we included a helpful utility script to show and save the image. Pipe the model results into the script. - -```sh -truss predict -d '{"prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"}' | python show.py -``` - -The output will be a dictionary with a key `data` mapping to a base64 encoded image. It's processed with this script: - -```python -import json -import base64 -import os, sys - -resp = sys.stdin.read() -image = json.loads(resp)["data"] -img=base64.b64decode(image) - -file_name = f'{image[-10:].replace("/", "")}.jpeg' -img_file = open(file_name, 'wb') -img_file.write(img) -img_file.close() -os.system(f'open {file_name}') -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://app.baseten.co/models/{MODEL_ID}/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" - }' -``` - -Again, the model will return a dictionary containing the base64-encoded image, which will need to be decoded and saved. diff --git a/stable-diffusion/sd-textual-inversion/README.md b/stable-diffusion/sd-textual-inversion/README.md deleted file mode 100644 index b2ce912c7..000000000 --- a/stable-diffusion/sd-textual-inversion/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Textual Inversion with Stable Diffusion - -The following example demonstrates how to use Stable Diffusion with -textual inversion embeddings. - -This truss combines concepts from: -1. [This colab](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/stable_conceptualizer_inference.ipynb#scrollTo=JkIeuLEfqi-g) which demonstrates how to load textual inversion embeddings from hugginface repos -2. [This diffusers issue](https://github.com/huggingface/diffusers/issues/3097#issuecomment-1516138396) which demonstrates how to load an embedding directly. diff --git a/stable-diffusion/sd-turbo/README.md b/stable-diffusion/sd-turbo/README.md deleted file mode 100644 index ebbb912a6..000000000 --- a/stable-diffusion/sd-turbo/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# SD Turbo - -SD Turbo is a fast text-to-image model built for real time image synthesis. It is a distilled version of Stable Diffusion 2.1. - -SD Turbo is a smaller model than [SDXL Turbo](https://github.com/basetenlabs/truss-examples/tree/main/stable-diffusion/sdxl-turbo). Comparatively, it achieves faster latency and tends to produce outputs of lower quality and prompt alignment. - -## Deploying SD Turbo - -First, clone this repository and navigate to the `sd-turbo` directory: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd sd-turbo -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `sd-turbo` as your working directory, deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -On a T4 with 16 GiB VRAM, we have observed that the SD Turbo mean inference time is < 0.5 seconds. - -## Invoking SD Turbo - -The SD Turbo truss takes in a JSON payload with two fields: -- `prompt` (required): Text describing the desired image. -- `num_steps` (optional, default = 1): Number of steps the model should iterate. Must be between 1-4. - -It returns a JSON object with a `result` field, which contains a generated image of size 512 x 512 encoded as a base 64 string. - -### Example usage - -The example code below invokes the SD Turbo truss, parses the image from the response, and saves the image. - -```python -from PIL import Image -from io import BytesIO -import base64 -import requests - -BASE64_PREAMBLE = "data:image/png;base64," - -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - -# Paste your model ID here. This can be grabbed from the Baseten UI or from the -# output of `truss push`. -MODEL_ID = "" -# Development model endpoint URL. To call the production deployment or another -# deployment, replace this with the desired endpoint URL from the Baseten UI. -MODEL_ENDPOINT = f"https://model-{MODEL_ID}.api.baseten.co/development/predict" - -# Paste your Baseten API key here. -API_KEY = "" -HEADERS = {"Authorization": f"Api-Key {API_KEY}"} - -resp = requests.post( - MODEL_ENDPOINT, - headers=HEADERS, - json={"prompt": "A tree in a field under the night sky"}, -) -resp = resp.json() -img = b64_to_pil(resp.get("result")) - -# Save the image. -img.save("sd_turbo_output.png") -``` diff --git a/stable-diffusion/sdxl-controlnet-canny/README.md b/stable-diffusion/sdxl-controlnet-canny/README.md deleted file mode 100644 index 83a720d54..000000000 --- a/stable-diffusion/sdxl-controlnet-canny/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Stable Diffusion XL + ControlNet Canny Truss - -This Truss uses Stable Diffusion XL and ControlNet with the Canny preprocessor to generate images guided by input image edges. The inputs are a prompt and an image. A Canny filter is applied to the image to generate a outline, which is then passed to SDXL with the prompt. - -![baseten_controlnet](baseten-logo.gif) - -## Deploying the Truss - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd sdxl-controlnet-canny-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `sdxl-controlnet-canny-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Using the model - -The model takes a JSON payload with two fields: - -- `prompt`: Text describing the desired image. -- `image`: Base64 encoded input image. - -It returns a JSON object with the `result` field containing the generated image. - -## Example Usage - -You can also invoke the SDXL + ControlNet model from Python using the `baseten` SDK: - -```python -import baseten -from PIL import Image -from io import BytesIO -import base64 - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -model = baseten.deployed_model_version_id("MODEL_VERSION_ID") # you can get this from the Baseten web UI - -image = Image.open("cat.png") -image_b64 = pil_to_b64() - -request = { - "prompt": "A painting of a cat", - "image": image_b64 -} - -response = model.predict(request) -``` - -The response will contain a base64 encoded image that you can save: - -```python -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - - -img = b64_to_pil(response["result"]) - -img.save('generated.png') -``` - -You can also invoke the model via REST API: - -```bash -curl -X POST "https://app.baseten.co/model_versions/VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H "Authorization: Api-Key {API_KEY}" \ - -d '{"prompt": "A painting of a cat", - "image": "data:image/png;base64,..."}' -``` - -The API will return a JSON response containing the generated image encoded in base64. diff --git a/stable-diffusion/sdxl-controlnet-depth/README.md b/stable-diffusion/sdxl-controlnet-depth/README.md deleted file mode 100644 index 3d66358b0..000000000 --- a/stable-diffusion/sdxl-controlnet-depth/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Stable Diffusion XL + ControlNet Depth Truss - -This Truss uses Stable Diffusion XL and ControlNet with the Depth preprocessor to generate images guided by the depth map of an input image. The inputs are a prompt and an image. - -## Deploying the Truss - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd sdxl-controlnet-depth-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `sdxl-controlnet-depth-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Using the model - -The model takes a JSON payload with two fields: - -- `prompt`: Text describing the desired image. -- `image`: Base64 encoded input image. - -It returns a JSON object with the `result` field containing the generated image. - -## Example Usage - -You can also invoke the SDXL + ControlNet model from Python using the `baseten` SDK: - -```python -import baseten -from PIL import Image -from io import BytesIO -import base64 - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -model = baseten.deployed_model_version_id("MODEL_VERSION_ID") # you can get this from the Baseten web UI - -image = Image.open("cat.png") -image_b64 = pil_to_b64() - -request = { - "prompt": "A painting of a cat", - "image": image_b64 -} - -response = model.predict(request) -``` - -The response will contain a base64 encoded image that you can save: - -```python -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - - -img = b64_to_pil(response["result"]) - -img.save('generated.png') -``` - -You can also invoke the model via REST API: - -```bash -curl -X POST "https://app.baseten.co/model_versions/VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H "Authorization: Api-Key {API_KEY}" \ - -d '{"prompt": "A painting of a cat", - "image": "data:image/png;base64,..."}' -``` - -The API will return a JSON response containing the generated image encoded in base64. diff --git a/stable-diffusion/sdxl-controlnet/README.md b/stable-diffusion/sdxl-controlnet/README.md deleted file mode 100644 index e2b621fda..000000000 --- a/stable-diffusion/sdxl-controlnet/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Stable Diffusion XL + ControlNet Truss - -This Truss uses Stable Diffusion XL and ControlNet to generate images guided by input image edges. The inputs are a prompt and an image. A Canny filter is applied to the image to generate a outline, which is then passed to SDXL with the prompt. - -![baseten_controlnet](baseten-logo.gif) - -## Deploying the Truss - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd sdxl-controlnet-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `sdxl-controlnet-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Using the model - -The model takes a JSON payload with two fields: - -- `prompt`: Text describing the desired image. -- `image`: Base64 encoded input image. - -It returns a JSON object with the `result` field containing the generated image. - -## Example Usage - -You can also invoke the SDXL + ControlNet model from Python using the `baseten` SDK: - -```python -import baseten - -model = baseten.deployed_model_version_id("MODEL_VERSION_ID") # you can get this from the Baseten web UI - -image = open("cat.png", "rb").read() -image_b64 = base64.b64encode(image).decode("utf-8") - -request = { - "prompt": "A painting of a cat", - "image": "data:image/png;base64," + image_b64 -} - -response = model.predict(request) -``` - -The response will contain a base64 encoded image that you can save: - -```python -import base64 - -img = base64.b64decode(response["result"]) - -with open("generated.png", "wb") as f: - f.write(img) -``` - -You can also invoke the model via REST API: - -```bash -curl -X POST "https://app.baseten.co/model_versions/VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H "Authorization: Api-Key {API_KEY}" \ - -d '{"prompt": "A painting of a cat", - "image": "data:image/png;base64,..."}' -``` - -The API will return a JSON response containing the generated image encoded in base64. diff --git a/stable-diffusion/sdxl-lightning/README.md b/stable-diffusion/sdxl-lightning/README.md deleted file mode 100644 index 91f932108..000000000 --- a/stable-diffusion/sdxl-lightning/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# SDXL Lightning Truss - -SDXL Turbo is a significant improvement over SDXL allowing for image generation in a single step while maintaining image quality. This model generates images with a latency of less than 1 second! - -This model is packaged using [Truss](https://trussml.com), the simplest way to serve AI/ML models in production. - -## Deploy SDXL Lightning - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion/sdxl-lightning -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-diffusion/sdxl-lightning` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -Model inference runs well on an A100. - -## Invoking SDXL Turbo - -This model only takes two inputs called `prompt` and `num_steps` and outputs a single image with the dimensions(512x512) encoded as a base 64 string. - -- `prompt` (required): Text describing the desired image - -It returns a JSON object with the `result` field containing the generated image as a base 64 string. - -Here is an example of how you can invoke the model using Python: - -```python -import base64 -import requests -import os - -# Replace the empty string with your model id below -model_id = "" -baseten_api_key = os.environ["BASETEN_API_KEY"] -BASE64_PREAMBLE = "data:image/png;base64," - -data = { - "prompt": "a picture of a rhino wearing a suit", -} - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/production/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=data -) - -# Get output image -res = res.json() -img_b64 = res.get("result") -img = base64.b64decode(img_b64) - -# Save the base64 string to a PNG -img_file = open("sdxl-output-1.png", "wb") -img_file.write(img) -img_file.close() -os.system("open sdxl-output-1.png") -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "An illustration of a rocket taking off an alien planet, vector art" - }' -``` diff --git a/stable-diffusion/sdxl-lora-swapping/README.md b/stable-diffusion/sdxl-lora-swapping/README.md deleted file mode 100644 index ee1fadadb..000000000 --- a/stable-diffusion/sdxl-lora-swapping/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Stable Diffusion XL with LoRA Swapping - -This Truss provides an example on how to hot-swap LoRAs with Stable Diffusion XL. This is useful when you have a bunch of customers with different fine-tunes but you want to use the same GPU instance. - -On Baseten, you can expect LoRA downloading + loading to take ~4s for a 120 MB LoRA and ~2s for a standard 20 MB LoRA. Generation time will be ~6s on an A100. - -## Deploying the Truss - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd sdxl-lora-swapping -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `sdxl-lora-swapping` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Using the model - -The model takes a JSON payload with two main fields: - -- `prompt`: Text describing the desired image. Make sure the prompt includes key phrases to activate the LoRA if needed. For example, many pixel style LoRAs require the words "pixel style" to be present in the prompt. -- `lora`: Dict with two keys, `repo_id` and `weights`. An example `lora` dict would like this: - -```lora.json -{ - "prompt": "pixel art, an baby giraffe", - "lora": {"repo_id": "nerijs/pixel-art-xl", "weights": "pixel-art-xl.safetensors"} -} -``` - -There are many ojptional fields including negative prompt and size that you can find in the `predict` function of the `model.py`. - -It returns a JSON object with the `result` field containing the generated image. - -## Example Usage - -You can use the Truss CLI to generate an image: - -``` -truss predict --model PRIMARY_MODEL_ID -f lora.json -``` - -Here, `lora.json` should contain the data payload for the request. - -You can also invoke the model via REST API: - -```bash -curl -X POST "https://app.baseten.co/model_versions/VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H "Authorization: Api-Key {API_KEY}" \ - -d '{"prompt": "pixel art, painting of a cat"}' -``` - -The API will return a JSON response containing the generated image encoded in base64. diff --git a/stable-diffusion/sdxl-lora/README.md b/stable-diffusion/sdxl-lora/README.md deleted file mode 100644 index 2d08ea240..000000000 --- a/stable-diffusion/sdxl-lora/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Stable Diffusion XL Truss with LoRA - -This is a [Truss](https://truss.baseten.co/) for Stable Diffusion XL using the `DiffusionPipeline` from the `diffusers` library. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Stable Diffusion XL with support for Low Rank Adaptation (LoRA). - -## Overview - -Stable Diffusion XL is an enhanced version of Stable Diffusion that generates 1024x1024 images. This Truss utilizes the `stabilityai/stable-diffusion-xl-base-1.0` model. - -This Truss also includes support for Low Rank Adaptation (LoRA). LoRA allows you to finetune Stable Diffusion on specific prompts to improve image quality and coherence. This Truss uses the `minimaxir/sdxl-wrong-lora` weights by default, which improves image quality by reducing malformed limbs and objects. - -## Deploying the Truss - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd sdxl-lora-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `sdxl-lora-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## API Documentation - -The API has one endpoint, `/predict`, which generates images. - -**Parameters:** - -- `prompt`: Input text prompt -- `size`: Image size, default 1024 -- `use_refiner`: Enable/disable secondary refiner model, default true -- `high_noise_frac`: Noise level for refiner model -- `num_inference_steps`: Number of denoising steps, default 30 - -**Example Request:** - -```json -{ - "prompt": "A beautiful painting of a fox in the forest", - "use_refiner": true, - "high_noise_frac": 0.8, - "num_inference_steps": 20 -} -``` - -**Returns:** Base64-encoded PNG image - -```json -{ - "result": "..." // base64 encoded image -} - -``` diff --git a/stable-diffusion/sdxl-lora/config.yaml b/stable-diffusion/sdxl-lora/config.yaml deleted file mode 100644 index 1a7d477d0..000000000 --- a/stable-diffusion/sdxl-lora/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_name: Stable Diffusion XL with LoRA -python_version: py311 -requirements: -- accelerate -- transformers -- safetensors -- opencv-python -- diffusers -resources: - accelerator: A10G - cpu: 3500m - memory: 20Gi - use_gpu: true -secrets: {} -system_packages: -- ffmpeg -- libsm6 -- libxext6 diff --git a/stable-diffusion/sdxl-turbo/README.md b/stable-diffusion/sdxl-turbo/README.md deleted file mode 100644 index f21d80ffa..000000000 --- a/stable-diffusion/sdxl-turbo/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# SDXL Turbo Truss - -SDXL Turbo is a significant improvement over SDXL allowing for image generation in a single step while maintaining image quality. This model generates images with a latency of less than 1 second! - -This model is packaged using [Truss](https://trussml.com), the simplest way to serve AI/ML models in production. - -![turbo](https://github.com/htrivedi99/truss-examples/assets/15642666/904aac3e-f06d-4161-b2c6-cf48522779ce) - - -## Deploy SDXL Turbo - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd sdxl-turbo -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `sdxl-turbo` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -Model inference runs well on an T4 with 16 GB of VRAM, with invocation time averaging ~1 second. - -## Invoking SDXL Turbo - -This model only takes two inputs called `prompt` and `num_steps` and outputs a single image with the dimensions(512x512) encoded as a base 64 string. - -- `prompt` (required): Text describing the desired image -- `num_steps` (optional): Number of steps the model should iterate. Must be between 1-4. - -It returns a JSON object with the `result` field containing the generated image as a base 64 string. - -Here is an example of how you can invoke the model using Python: - -```python -BASE64_PREAMBLE = "data:image/png;base64," -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - - -headers = {"Authorization": f"Api-Key "} -resp = requests.post( - "https://model-.api.baseten.co/development/predict", - headers=headers, - json={"prompt": "An illustration of a rocket taking off an alien planet, vector art"}, -) -resp = resp.json() -img = b64_to_pil(resp.get("result")) -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "An illustration of a rocket taking off an alien planet, vector art" - }' -``` diff --git a/stable-diffusion/stable-diffusion-3-medium/README.md b/stable-diffusion/stable-diffusion-3-medium/README.md deleted file mode 100644 index 075dd0a18..000000000 --- a/stable-diffusion/stable-diffusion-3-medium/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# Stable Diffusion 3 Medium Truss - -This is a truss for the brand new Stable Diffusion 3 Medium model. - -## Deploy Stable Diffusion 3 - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion/stable-diffusion-3-medium -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-diffusion-3-medium` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using Stable Diffusion through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -## Invoking Stable Diffusion 3 - -The output will be a dictionary with a key `data` mapping to a base64 encoded image. It's processed with this script: - -```python -import requests -import os -import base64 -from PIL import Image -from io import BytesIO - -# Replace the empty string with your model id below -model_id = "" -baseten_api_key = os.environ["BASETEN_API_KEY"] -BASE64_PREAMBLE = "data:image/png;base64," - -# Function used to convert a base64 string to a PIL image -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - -data = { - "prompt": "a little boy looking through a large magical portal, the boy sees a futuristic human civilization in that portal, extremely detailed, trending on artstation, 8k" -} - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/production/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=data -) - -# Get output image -res = res.json() -output = res.get("data") - -# Convert the base64 model output to an image -img = b64_to_pil(output) -img.save("output_image.png") -os.system("open output_image.png") -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://model-{model_id}.api.baseten.co/production/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "A tree in a field under the night sky" - }' -``` - -Again, the model will return a dictionary containing the base64-encoded image, which will need to be decoded and saved. diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/README.md b/stable-diffusion/stable-diffusion-inpainting-trt/README.md deleted file mode 100644 index 6d8d306f6..000000000 --- a/stable-diffusion/stable-diffusion-inpainting-trt/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# Stable Diffusion Inpainting TensorRT - -This is a [Truss](https://truss.baseten.co/) for Stable Diffusion v1.5 Inpainting. It has been optimized to increase performance using TensorRT. - -## Deploy Stable Diffusion Inpainting TensorRT - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion/stable-diffusion-inpainting-trt -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-diffusion-inpainting-trt` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using Stable Diffusion through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -## Invoking Stable Diffusion Inpainting TensorRT - -The model accepts a few inputs: -- __prompt__(required): Text describing the output image. -- __negative_prompt__(optional): Text used to steer the model away from undesired output. -- __image__(required): The input image used for inpainting in the form of a base64 string. The image should be 512 x 512 px in size. -- __mask__(required): The an image representing the mask or the area that you want stable diffusion to generate over. It should also be a base64 string and the same dimensions as the input image. - -```python -from PIL import Image -import base64 -import requests - -BASE64_PREAMBLE = "data:image/png;base64," - -def pil_to_b64(pil_img): - buffered = BytesIO() - pil_img.save(buffered, format="PNG") - img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") - return img_str - -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - - -data = { - "prompt": "A tiger", - "image": pil_to_b64(Image.open("/path/to/image/dog.png")), - "mask": pil_to_b64(Image.open("/path/to/mask/mask.png")) -} - -headers = {"Authorization": "Api-Key "} - -res = requests.post( - "https://model-.api.baseten.co/development/predict", - headers=headers, - json=data, -) - -res = res.json() -outputs = res.get("outputs") - -for out in outputs: - img = b64_to_pil(out) - img.show() -``` diff --git a/stable-diffusion/stable-diffusion-inpainting-trt/config.yaml b/stable-diffusion/stable-diffusion-inpainting-trt/config.yaml deleted file mode 100644 index 7cde4606e..000000000 --- a/stable-diffusion/stable-diffusion-inpainting-trt/config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_name: Stable Diffusion Inpainting TRT -python_version: py310 -requirements: [] -requirements_file: requirements.txt -resources: - accelerator: A10G - use_gpu: true -secrets: {} -system_packages: [] diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md b/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md deleted file mode 100644 index 400d9f38c..000000000 --- a/stable-diffusion/stable-diffusion-xl-1.0-trt-h100/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Stable Diffusion XL TensorRT Truss for H100 - -Stable Diffusion XL 1.0 is the largest, most capable open-source image generation model of its kind. This README covers deploying and invoking this model. - -This model is packaged using [Truss](https://trussml.com), the simplest way to serve AI/ML models in production. - -## Deploy Stable Diffusion XL TensorRT - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion/stable-diffusion-xl-1.0-trt-h100 -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-diffusion-xl-1.0-trt-h100` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using SDXL through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -### Hardware notes - -Running inference on an H100 cuts invocation time to <2 seconds. - -## Invoking Stable Diffusion XL TensorRT - -Stable Diffusion XL TensorRT returns an image in Base 64, which is not super useful as a string in your terminal. So we included a helpful utility script to show and save the image. Pipe the model results into the script. - -```sh -truss predict -d '{"prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"}' | python show.py -``` - -The output will be a dictionary with a key `data` mapping to a base64 encoded image. It's processed with this script: - -```python -import json -import base64 -import os, sys - -resp = sys.stdin.read() -image = json.loads(resp)["data"] -img=base64.b64decode(image) - -file_name = f'{image[-10:].replace("/", "")}.jpeg' -img_file = open(file_name, 'wb') -img_file.write(img) -img_file.close() -os.system(f'open {file_name}') -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://app.baseten.co/models/{MODEL_ID}/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" - }' -``` - -Again, the model will return a dictionary containing the base64-encoded image, which will need to be decoded and saved. diff --git a/stable-diffusion/stable-diffusion-xl-1.0-trt/README.md b/stable-diffusion/stable-diffusion-xl-1.0-trt/README.md deleted file mode 100644 index 23fe732c9..000000000 --- a/stable-diffusion/stable-diffusion-xl-1.0-trt/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Stable Diffusion XL TensorRT Truss - -Stable Diffusion XL 1.0 is the largest, most capable open-source image generation model of its kind. This README covers deploying and invoking this model. - -This model is packaged using [Truss](https://trussml.com), the simplest way to serve AI/ML models in production. - -## Deploy Stable Diffusion XL TensorRT - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion/stable-diffusion-xl-1.0-trt -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-diffusion-xl-1.0-trt` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using SDXL through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -### Hardware notes - -Running inference on an A100 cuts invocation time to ~3.5 seconds. - -## Invoking Stable Diffusion XL TensorRT - -Stable Diffusion XL TensorRT returns an image in Base 64, which is not super useful as a string in your terminal. So we included a helpful utility script to show and save the image. Pipe the model results into the script. - -```sh -truss predict -d '{"prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"}' | python show.py -``` - -The output will be a dictionary with a key `data` mapping to a base64 encoded image. It's processed with this script: - -```python -import json -import base64 -import os, sys - -resp = sys.stdin.read() -image = json.loads(resp)["data"] -img=base64.b64decode(image) - -file_name = f'{image[-10:].replace("/", "")}.jpeg' -img_file = open(file_name, 'wb') -img_file.write(img) -img_file.close() -os.system(f'open {file_name}') -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://app.baseten.co/models/{MODEL_ID}/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" - }' -``` - -Again, the model will return a dictionary containing the base64-encoded image, which will need to be decoded and saved. diff --git a/stable-diffusion/stable-diffusion-xl-1.0/README.md b/stable-diffusion/stable-diffusion-xl-1.0/README.md deleted file mode 100644 index 39a1d670f..000000000 --- a/stable-diffusion/stable-diffusion-xl-1.0/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# Stable Diffusion XL Truss - -Stable Diffusion XL 1.0 is the largest, most capable open-source image generation model of its kind. This README covers deploying and invoking this model. - -This model is packaged using [Truss](https://trussml.com), the simplest way to serve AI/ML models in production. - -## Deploy Stable Diffusion XL - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion-xl-1.0 -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-diffusion-xl-1.0` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using SDXL through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -### Hardware notes - -Model inference runs well on an A10 with 24 GB of VRAM, with invocation time averaging ~8 seconds. If speed is essential, running inference on an A100 cuts invocation time to ~4 seconds. - -## Invoking Stable Diffusion XL - -Stable Diffusion XL returns an image in Base 64, which is not super useful as a string in your terminal. So we included a helpful utility script to show and save the image. Pipe the model results into the script. - -```sh -truss predict -d '{"prompt": "A tree in a field under the night sky"}' | python show.py -``` - -The output will be a dictionary with a key `data` mapping to a base64 encoded image. It's processed with this script: - -```python -import json -import base64 -import os, sys - -resp = sys.stdin.read() -image = json.loads(resp)["data"] -img=base64.b64decode(image) - -file_name = f'{image[-10:].replace("/", "")}.jpeg' -img_file = open(file_name, 'wb') -img_file.write(img) -img_file.close() -os.system(f'open {file_name}') -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://app.baseten.co/models/MODEL_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "A tree in a field under the night sky", - "use_refiner": True - }' -``` - -Again, the model will return a dictionary containing the base64-encoded image, which will need to be decoded and saved. - -Here is a complete example of invoking this model in Python: - -## Model Input - -```python -import requests -import os -import base64 -from PIL import Image -from io import BytesIO - -# Replace the empty string with your model id below -model_id = "" -baseten_api_key = os.environ["BASETEN_API_KEY"] -BASE64_PREAMBLE = "data:image/png;base64," - -# Function used to convert a base64 string to a PIL image -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - -data = { - "prompt": "a little boy looking through a large magical portal, the boy sees a futuristic human civilization in that portal, extremely detailed, trending on artstation, 8k" -} - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/production/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=data -) - -# Get output image -res = res.json() -output = res.get("data") - -# Convert the base64 model output to an image -img = b64_to_pil(output) -img.save("output_image.png") -os.system("open output_image.png") -``` - -## Model Output -```json -{"data": "iVBORw0KGgoAAAANSUhEUgAABAAAAAQA..."} -``` - -Here is the output image for the prompt shown in the request above: -![a_little_boy_looking_through_a_large_magical_portal,_the_boy_sees_a_futuristic_human_civilization_in](https://github.com/htrivedi99/truss-examples/assets/15642666/c534c752-29cb-4da8-b24e-fda6bef5876c) - -Here's another example using more SDXL configurations: - -```python -import requests -import os -import base64 -from PIL import Image -from io import BytesIO - -# Replace the empty string with your model id below -model_id = "" -baseten_api_key = os.environ["BASETEN_API_KEY"] -BASE64_PREAMBLE = "data:image/png;base64," - -# Function used to convert a base64 string to a PIL image -def b64_to_pil(b64_str): - return Image.open(BytesIO(base64.b64decode(b64_str.replace(BASE64_PREAMBLE, "")))) - -data = { - "prompt": "Extremely detailed and intricate scene of baby phoenix hatchling cuddled up resting on a pile of ashes surrounded by fire and smoke, rays of sunlight shine on the phoenix, in the background is a dense dark forest, settings: f/8 aperture, full shot, hyper realistic, 4k", - "negative_prompt": "worst quality, low quality", - "width": 1248, - "height": 832, - "num_inference_steps": 35, - "use_refiner": False, - "scheduler": "DPM++ 2M", - "guidance_scale": 14 -} - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/production/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=data -) - -# Get output image -res = res.json() -output = res.get("data") - -# Convert the base64 model output to an image -img = b64_to_pil(output) -img.save("output_image.png") -os.system("open output_image.png") -``` - -Here is the model output: -```json -{"data": "iVBORw0KGgoAAAANSUhEUgAABAAAAAQA..."} -``` - -This is the output image for the prompt above: -![Extremely_detailed_and_intricate_scene_of_baby_phoenix_hatchling_cuddled_up_resting_on_a_pile_of_ash](https://github.com/htrivedi99/truss-examples/assets/15642666/1fbf004a-9741-4c83-90a8-9e7d51dfd4e2) diff --git a/stable-diffusion/stable-diffusion/README.md b/stable-diffusion/stable-diffusion/README.md deleted file mode 100644 index 55ad20941..000000000 --- a/stable-diffusion/stable-diffusion/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# Stable Diffusion Truss - -This is a [Truss](https://truss.baseten.co/) for Stable Diffusion v2.1 using the `StableDiffusionPipeline` from the `diffusers` library. This README will walk you through how to deploy this Truss on Baseten to get your own instance of the Stable Diffusion. - -## Deploy Stable Diffusion - -First, clone this repository: - -``` -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-diffusion-truss` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using Stable Diffusion through the Baseten platform! Navigate to the Baseten UI to watch the model build and deploy and invoke it via the REST API. - -## Invoking Stable Diffusion - -Stable Diffusion returns an image in Base 64, which is not super useful as a string in your terminal. So we included a helpful utility script to show and save the image. Pipe the model results into the script. - -```sh -truss predict -d '{"prompt": "A tree in a field under the night sky"}' | python show.py -``` - -The output will be a dictionary with a key `data` mapping to a base64 encoded image. It's processed with this script: - -```python -import json -import base64 -import os, sys - -resp = sys.stdin.read() -image = json.loads(resp)["data"] -img=base64.b64decode(image) - -file_name = f'{image[-10:].replace("/", "")}.jpeg' -img_file = open(file_name, 'wb') -img_file.write(img) -img_file.close() -os.system(f'open {file_name}') -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST "https://app.baseten.co/models/MODEL_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "A tree in a field under the night sky" - }' -``` - -Again, the model will return a dictionary containing the base64-encoded image, which will need to be decoded and saved. - -### Stable Diffusion API documentation - -This section provides an overview of the Stable Diffusion API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate images based on the provided parameters. - -#### API route: `predict` - -The predict route is the primary method for generating images based on a given set of parameters. It takes several parameters: - -- **prompt**: The input text you'd like to generate an image for -- **scheduler**: (optional, default: DDIM) The scheduler used for the diffusion process. Choose from: "ddim", "dpm", "euler", "lms", or "pndm". -- **seed**: (optional) A random seed for deterministic results. If not provided, a random seed will be generated. -- **negative_prompt**: (optional) A string representing the negative prompt, or prompts that indicate what you don't want to generate. - -The API also supports passing any parameter supported by Diffuser's `StableDiffusionPipeline`. diff --git a/stable-diffusion/stable-video-diffusion/README.md b/stable-diffusion/stable-video-diffusion/README.md deleted file mode 100644 index fb9ceb9f5..000000000 --- a/stable-diffusion/stable-video-diffusion/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Stable Diffusion Video - -This Truss allows you to create small videos ( < 10 seconds) using the [Stable Video Diffusion model](https://stability.ai/news/stable-video-diffusion-open-ai-video-model). The model only requires a single image as input and converts it into a video. - - -https://github.com/htrivedi99/truss-examples/assets/15642666/53d3ea1f-952e-4224-8bd4-3ff95eeac8c0 - - -## Deploying the Truss - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd stable-diffusion/stable-video-diffusion -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `stable-video-diffusion` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Using the model - -The model takes a JSON payload with one required field and 4 optional fields: - -- `image` (required): The input image as a base64 string. This model was trained on images that are 1024β€ŠΓ—β€Š576 in size. Other image dimensions work as well for the most part, but to get the best results using 1024β€ŠΓ—β€Š576 images is recommended. -- `num_frames` (optional) - The total number of frames in the output clip -- `num_steps` (optional) - Steps takes at each iteration -- `fps` (optional) - Frames per second for the clip -- `decoding_t` (optional) - The number of frames decoded per second. This number cannot be greater than 10 as the GPU runs out of memory at that point. - -It returns a JSON object with the `output` key containing the generated video as a base64 string. - -## Example Usage - -Here is how you can invoke the model using Python: - -```python -def base64_to_mp4(base64_string, output_file_path): - binary_data = base64.b64decode(base64_string) - with open(output_file_path, "wb") as output_file: - output_file.write(binary_data) - - -def image_to_base64(file_path): - with open(file_path, "rb") as image_file: - binary_data = image_file.read() - base64_data = base64.b64encode(binary_data) - base64_string = base64_data.decode("utf-8") - - return base64_string - -data = { - "image": image_to_base64("path/to/image/cheetah.jpeg"), - "num_frames": 14, - "fps": 10, - "decoding_t": 5 -} -headers = {"Authorization": f"Api-Key "} -res = requests.post("https://model-.api.baseten.co/development/predict", headers=headers, json=data) -res = res.json() - -base64_output = res.get("output") -base64_to_mp4(base64_output, "output_video.mp4") -``` - -You can also invoke the model via REST API: - -```bash -curl -X POST "https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H "Authorization: Api-Key {BASETEN-API-KEY}" \ - -d '{"image": ""}' -``` - -For inspiration, here are some sample input images you can use: - -![image1](sample_images/cheetah.jpeg) -![image2](sample_images/racecar.jpeg) diff --git a/text-embeddings-inference/README.md b/text-embeddings-inference/README.md deleted file mode 100644 index a52303ffa..000000000 --- a/text-embeddings-inference/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# Notice -This section has moved to [jina-ai/jina-embeddings-v2-base-en-TEI](https://github.com/basetenlabs/truss-examples/tree/main/11-embeddings-reranker-classification-tensorrt) with an overview over fast embeddings. - -# Text Embeddings Inference Truss - -This is a Trussless Customer Server example to deploy [text-embeddings-inference](https://github.com/huggingface/text-embeddings-inference), a high performance server that handles text-embeddings, ranranking and classification models as api. - -## Deployment - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. [Required for gated model] Retrieve your Hugging Face token from the [settings](https://huggingface.co/settings/tokens). Set your Hugging Face token as a Baseten secret [here](https://app.baseten.co/settings/secrets) with the key `hf_access_key`. - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd text-embeddings-inference -``` - -With `text-embeddings-inference` as your working directory, you can deploy the model with the following command, paste your Baseten API key if prompted. - -```sh -truss push --publish -``` - -## Performance Optimization: - -The config.yaml contains a couple of variables that can be tuned, depending on: -- which GPU is used -- which model is deployed -- how many concurrent requests users are sending - -The deployment example is for Bert-large and a Nvidia-L4. Bert-large has a maxiumum sequence length of 512 tokens per sentence. -For Bert-large architecture & the L4, there are marginal gains above a batch-size of 16000 tokens. - -### Concurrent requests -``` ---max-concurrent-requests 40 -# and -runtime: - predict_concurrency : 40 -``` -The following set the number of parallel `post` requests. -In this case we allow 40 parallel requests to be handled per replica & should allow to batch requests from multiple users together, reaching high token counts. Potentially 40 single parallel requests with one sequence each could fully utilize the GPU. `1*40*512=20480` - - -### Tokens per batch -``` ---max-batch-tokens 32768 -``` - -This number of total tokens in a batch. For embedding models, this will determine the VRAM usage. -As most of TEI's models are implemented with `nested` attention implementation, `32768 tokens` could mean `64 sentence with 512 tokens` or `512 sentences with 64 tokens`. While the first will take slightly longer to compute, the peak VRAM usage will stay roughly the same. For `llama` or `mistral` based `7b` embedding models, we recommend setting it a lower setting e.g. -``` ---max-batch-tokens 8192 -``` - -### Client batch size -``` ---max-client-batch-size 32 -``` -Client match size determines the number of sentences in a single request. -Increase if clients cannot send multiple concurrent requests, or if clients require to larger requests size. - -### Endpoint, Model Selection, and OpenAPI -Change to /rerank or /predict if you want to use the rerank or predict endpoint. -Embedding model. -Example supported models: https://huggingface.co/models?pipeline_tag=feature-extraction&other=text-embeddings-inference&sort=trending -```yaml - predict_endpoint: /v1/embeddings -``` -Rerank model. -Example models https://huggingface.co/models?pipeline_tag=text-classification&other=text-embeddings-inference&sort=trending -```yaml - predict_endpoint: /rerank -``` -Classification model: -Example classification model: https://huggingface.co/SamLowe/roberta-base-go_emotions -```yaml - predict_endpoint: /predict -``` - -## Call your model - -### curl - -```bash -curl -X POST https://model-xxx.api.baseten.co/development/predict \ - -H "Authorization: Api-Key YOUR_API_KEY" \ - -d '{"input": "text string"}' -``` - - -### request python library - -```python -import os -import requests - -resp = requests.post( - "https://model-xxx.api.baseten.co/environments/production/predict", - headers={"Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}"}, - json={"input": ["text string", "second string"]}, -) - -print(resp.json()) -``` - - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/tutorials/README.md b/tutorials/README.md new file mode 100644 index 000000000..16d553c2f --- /dev/null +++ b/tutorials/README.md @@ -0,0 +1,23 @@ +# Tutorials + +Step-by-step guides for learning how to build, configure, and deploy models with Truss. Each tutorial focuses on a single concept and includes a complete, deployable example. + +| Directory | Description | +|-----------|-------------| +| [getting-started-bert](getting-started-bert/) | Introductory tutorial packaging a BERT model for deployment | +| [llm-basics](llm-basics/) | Serve a large language model with basic inference | +| [llm-streaming](llm-streaming/) | Stream token-by-token responses from an LLM | +| [image-generation](image-generation/) | Generate images using a diffusion model | +| [speech-to-text](speech-to-text/) | Transcribe audio input to text | +| [cached-weights](cached-weights/) | Use cached model weights for faster cold starts | +| [dynamic-batching](dynamic-batching/) | Batch incoming requests for higher throughput | +| [private-huggingface](private-huggingface/) | Load models from a private HuggingFace repository | +| [system-packages](system-packages/) | Install system-level dependencies in your Truss environment | + +## Deploying + +Each tutorial can be deployed to Baseten with: + +```bash +truss push +``` diff --git a/tutorials/cached-weights/README.md b/tutorials/cached-weights/README.md new file mode 100644 index 000000000..8f5f47bcd --- /dev/null +++ b/tutorials/cached-weights/README.md @@ -0,0 +1,32 @@ +# Llama with Cached Weights + +A tutorial example showing how to deploy Llama with Cached Weights on Baseten. + +| Property | Value | +|----------|-------| +| Model | [NousResearch/Llama-2-7b-chat-hf](https://huggingface.co/NousResearch/Llama-2-7b-chat-hf) | +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "What is the meaning of life?" +}' +``` + +## Configuration highlights + +- Model cache: **volume-mounted** for fast cold starts diff --git a/06-high-performance-cached-weights/config.yaml b/tutorials/cached-weights/config.yaml similarity index 94% rename from 06-high-performance-cached-weights/config.yaml rename to tutorials/cached-weights/config.yaml index 4748b9b55..4e9bbb6a1 100644 --- a/06-high-performance-cached-weights/config.yaml +++ b/tutorials/cached-weights/config.yaml @@ -2,6 +2,7 @@ # # The `config.yaml` file is where you need to include the changes to # actually cache the weights at build time. +description: "A tutorial example showing how to deploy Llama with Cached Weights on Baseten." environment_variables: {} external_package_dirs: [] model_metadata: diff --git a/06-high-performance-cached-weights/doc.yaml b/tutorials/cached-weights/doc.yaml similarity index 100% rename from 06-high-performance-cached-weights/doc.yaml rename to tutorials/cached-weights/doc.yaml diff --git a/whisper/faster-whisper-v3/model/__init__.py b/tutorials/cached-weights/model/__init__.py similarity index 100% rename from whisper/faster-whisper-v3/model/__init__.py rename to tutorials/cached-weights/model/__init__.py diff --git a/06-high-performance-cached-weights/model/model.py b/tutorials/cached-weights/model/model.py similarity index 100% rename from 06-high-performance-cached-weights/model/model.py rename to tutorials/cached-weights/model/model.py diff --git a/07-high-performance-dynamic-batching/.gitignore b/tutorials/dynamic-batching/.gitignore similarity index 100% rename from 07-high-performance-dynamic-batching/.gitignore rename to tutorials/dynamic-batching/.gitignore diff --git a/07-high-performance-dynamic-batching/.truss_ignore b/tutorials/dynamic-batching/.truss_ignore similarity index 100% rename from 07-high-performance-dynamic-batching/.truss_ignore rename to tutorials/dynamic-batching/.truss_ignore diff --git a/tutorials/dynamic-batching/README.md b/tutorials/dynamic-batching/README.md new file mode 100644 index 000000000..1f6cc877c --- /dev/null +++ b/tutorials/dynamic-batching/README.md @@ -0,0 +1,33 @@ +# TRT Whisper - Dynamic Batching + +A tutorial example showing how to deploy TRT Whisper - Dynamic Batching on Baseten. + +| Property | Value | +|----------|-------| +| Model | [baseten/trtllm-whisper-a10g-large-v2-1](https://huggingface.co/baseten/trtllm-whisper-a10g-large-v2-1) | +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"audio": ""}' +``` + +## Configuration highlights + +- Base image: `baseten/trtllm-server:r23.12_baseten_v0.9.0.dev2024022000` +- Model cache: **volume-mounted** for fast cold starts +- Predict concurrency: **256** +- System packages: `python3.10-venv, ffmpeg` diff --git a/07-high-performance-dynamic-batching/config.yaml b/tutorials/dynamic-batching/config.yaml similarity index 82% rename from 07-high-performance-dynamic-batching/config.yaml rename to tutorials/dynamic-batching/config.yaml index fae83526a..dfdee9dd6 100644 --- a/07-high-performance-dynamic-batching/config.yaml +++ b/tutorials/dynamic-batching/config.yaml @@ -1,6 +1,9 @@ +description: "A tutorial example showing how to deploy TRT Whisper - Dynamic Batching on Baseten." base_image: image: baseten/trtllm-server:r23.12_baseten_v0.9.0.dev2024022000 python_executable_path: /usr/bin/python3 +model_metadata: + example_model_input: {"audio": "UklGRiQAAABXQVZFZm10IBAAAAABAAEAgD4AAIA+AAABAAgAZGF0YQAAAAA="} model_name: TRT Whisper - Dynamic Batching python_version: py311 requirements: diff --git a/whisper/whisper-streaming/model/__init__.py b/tutorials/dynamic-batching/model/__init__.py similarity index 100% rename from whisper/whisper-streaming/model/__init__.py rename to tutorials/dynamic-batching/model/__init__.py diff --git a/07-high-performance-dynamic-batching/model/model.py b/tutorials/dynamic-batching/model/model.py similarity index 100% rename from 07-high-performance-dynamic-batching/model/model.py rename to tutorials/dynamic-batching/model/model.py diff --git a/whisper/whisper-torchserve/model/__init__.py b/tutorials/dynamic-batching/packages/__init__.py similarity index 100% rename from whisper/whisper-torchserve/model/__init__.py rename to tutorials/dynamic-batching/packages/__init__.py diff --git a/07-high-performance-dynamic-batching/packages/run.py b/tutorials/dynamic-batching/packages/run.py similarity index 100% rename from 07-high-performance-dynamic-batching/packages/run.py rename to tutorials/dynamic-batching/packages/run.py diff --git a/07-high-performance-dynamic-batching/packages/tokenizer.py b/tutorials/dynamic-batching/packages/tokenizer.py similarity index 100% rename from 07-high-performance-dynamic-batching/packages/tokenizer.py rename to tutorials/dynamic-batching/packages/tokenizer.py diff --git a/07-high-performance-dynamic-batching/packages/whisper_utils.py b/tutorials/dynamic-batching/packages/whisper_utils.py similarity index 100% rename from 07-high-performance-dynamic-batching/packages/whisper_utils.py rename to tutorials/dynamic-batching/packages/whisper_utils.py diff --git a/07-high-performance-dynamic-batching/test.py b/tutorials/dynamic-batching/test.py similarity index 100% rename from 07-high-performance-dynamic-batching/test.py rename to tutorials/dynamic-batching/test.py diff --git a/tutorials/getting-started-bert/README.md b/tutorials/getting-started-bert/README.md new file mode 100644 index 000000000..267716909 --- /dev/null +++ b/tutorials/getting-started-bert/README.md @@ -0,0 +1,31 @@ +# bert + +A tutorial example showing how to deploy bert on Baseten. + +| Property | Value | +|----------|-------| +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | CPU | +| Python | py310 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Hello my name is {MASK}" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/01-getting-started-bert/config.yaml b/tutorials/getting-started-bert/config.yaml similarity index 92% rename from 01-getting-started-bert/config.yaml rename to tutorials/getting-started-bert/config.yaml index e833ba318..0aa3d2768 100644 --- a/01-getting-started-bert/config.yaml +++ b/tutorials/getting-started-bert/config.yaml @@ -9,9 +9,11 @@ # # In this section, we can define basic metadata about the model, # such as the name, and the Python version to build with. +description: "A tutorial example showing how to deploy bert on Baseten." model_name: bert python_version: py310 model_metadata: + repo_id: "google-bert/bert-base-uncased" example_model_input: { "text": "Hello my name is {MASK}" } diff --git a/01-getting-started-bert/doc.yaml b/tutorials/getting-started-bert/doc.yaml similarity index 100% rename from 01-getting-started-bert/doc.yaml rename to tutorials/getting-started-bert/doc.yaml diff --git a/whisper/whisper-truss/model/__init__.py b/tutorials/getting-started-bert/model/__init__.py similarity index 100% rename from whisper/whisper-truss/model/__init__.py rename to tutorials/getting-started-bert/model/__init__.py diff --git a/01-getting-started-bert/model/model.py b/tutorials/getting-started-bert/model/model.py similarity index 100% rename from 01-getting-started-bert/model/model.py rename to tutorials/getting-started-bert/model/model.py diff --git a/tutorials/image-generation/README.md b/tutorials/image-generation/README.md new file mode 100644 index 000000000..bf9e0f088 --- /dev/null +++ b/tutorials/image-generation/README.md @@ -0,0 +1,32 @@ +# Stable Diffusion XL + +A tutorial example showing how to deploy Stable Diffusion XL on Baseten. + +| Property | Value | +|----------|-------| +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A tree in a field under the night sky", + "use_refiner": true +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg, libsm6, libxext6` diff --git a/04-image-generation/config.yaml b/tutorials/image-generation/config.yaml similarity index 94% rename from 04-image-generation/config.yaml rename to tutorials/image-generation/config.yaml index 3472543ce..805ebcbe6 100644 --- a/04-image-generation/config.yaml +++ b/tutorials/image-generation/config.yaml @@ -2,9 +2,11 @@ # # Running SDXL requires a handful of Python libraries, including # diffusers, transformers, and others. +description: "A tutorial example showing how to deploy Stable Diffusion XL on Baseten." environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: stabilityai/stable-diffusion-xl-base-1.0 example_model_input: { "prompt": "A tree in a field under the night sky", "use_refiner": true } model_name: Stable Diffusion XL diff --git a/04-image-generation/doc.yaml b/tutorials/image-generation/doc.yaml similarity index 100% rename from 04-image-generation/doc.yaml rename to tutorials/image-generation/doc.yaml diff --git a/whisper/whisper-v3-truss-base64/model/__init__.py b/tutorials/image-generation/model/__init__.py similarity index 100% rename from whisper/whisper-v3-truss-base64/model/__init__.py rename to tutorials/image-generation/model/__init__.py diff --git a/04-image-generation/model/model.py b/tutorials/image-generation/model/model.py similarity index 100% rename from 04-image-generation/model/model.py rename to tutorials/image-generation/model/model.py diff --git a/tutorials/llm-basics/README.md b/tutorials/llm-basics/README.md new file mode 100644 index 000000000..8abebb04d --- /dev/null +++ b/tutorials/llm-basics/README.md @@ -0,0 +1,33 @@ +# Mistral 7B + +A tutorial example showing how to deploy Mistral 7B on Baseten. + +| Property | Value | +|----------|-------| +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py311 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "What is the meaning of life?" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/02-llm/config.yaml b/tutorials/llm-basics/config.yaml similarity index 88% rename from 02-llm/config.yaml rename to tutorials/llm-basics/config.yaml index 3e03fb42c..8c3fb3d37 100644 --- a/02-llm/config.yaml +++ b/tutorials/llm-basics/config.yaml @@ -2,9 +2,11 @@ # # Running Mistral 7B requires a few libraries, such as # `torch`, `transformers` and a couple others. +description: "A tutorial example showing how to deploy Mistral 7B on Baseten." environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "mistralai/Mistral-7B-v0.1" example_model_input: {"prompt": "What is the meaning of life?"} model_name: Mistral 7B python_version: py311 diff --git a/02-llm/doc.yaml b/tutorials/llm-basics/doc.yaml similarity index 100% rename from 02-llm/doc.yaml rename to tutorials/llm-basics/doc.yaml diff --git a/whisper/whisper-v3-truss/model/__init__.py b/tutorials/llm-basics/model/__init__.py similarity index 100% rename from whisper/whisper-v3-truss/model/__init__.py rename to tutorials/llm-basics/model/__init__.py diff --git a/02-llm/model/model.py b/tutorials/llm-basics/model/model.py similarity index 100% rename from 02-llm/model/model.py rename to tutorials/llm-basics/model/model.py diff --git a/tutorials/llm-streaming/README.md b/tutorials/llm-streaming/README.md new file mode 100644 index 000000000..06cffa388 --- /dev/null +++ b/tutorials/llm-streaming/README.md @@ -0,0 +1,30 @@ +# LLM with Streaming + +A tutorial example showing how to deploy LLM with Streaming on Baseten. + +| Property | Value | +|----------|-------| +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | A10G | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "what is the meaning of life" +}' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/03-llm-with-streaming/config.yaml b/tutorials/llm-streaming/config.yaml similarity index 81% rename from 03-llm-with-streaming/config.yaml rename to tutorials/llm-streaming/config.yaml index a6ca9504b..0a5958c2c 100644 --- a/03-llm-with-streaming/config.yaml +++ b/tutorials/llm-streaming/config.yaml @@ -2,8 +2,10 @@ # # Running Falcon 7B requires torch, transformers, # and a few other related libraries. +description: "A tutorial example showing how to deploy LLM with Streaming on Baseten." model_name: "LLM with Streaming" model_metadata: + repo_id: "tiiuae/falcon-7b-instruct" example_model_input: {"prompt": "what is the meaning of life"} requirements: - torch==2.0.1 diff --git a/03-llm-with-streaming/doc.yaml b/tutorials/llm-streaming/doc.yaml similarity index 100% rename from 03-llm-with-streaming/doc.yaml rename to tutorials/llm-streaming/doc.yaml diff --git a/whisper/whisperx-truss/model/__init__.py b/tutorials/llm-streaming/model/__init__.py similarity index 100% rename from whisper/whisperx-truss/model/__init__.py rename to tutorials/llm-streaming/model/__init__.py diff --git a/03-llm-with-streaming/model/model.py b/tutorials/llm-streaming/model/model.py similarity index 100% rename from 03-llm-with-streaming/model/model.py rename to tutorials/llm-streaming/model/model.py diff --git a/tutorials/private-huggingface/README.md b/tutorials/private-huggingface/README.md new file mode 100644 index 000000000..cdec9e4bc --- /dev/null +++ b/tutorials/private-huggingface/README.md @@ -0,0 +1,31 @@ +# private-model + +A tutorial example showing how to deploy private-model on Baseten. + +| Property | Value | +|----------|-------| +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | CPU | +| Python | py39 | + +## Deploy + +> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying. + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '"It is a [MASK] world"' +``` + +## Configuration highlights + +- Engine: **Custom (Truss)** diff --git a/09-private-huggingface/config.yaml b/tutorials/private-huggingface/config.yaml similarity index 83% rename from 09-private-huggingface/config.yaml rename to tutorials/private-huggingface/config.yaml index 8581f513e..3be8bf95c 100644 --- a/09-private-huggingface/config.yaml +++ b/tutorials/private-huggingface/config.yaml @@ -3,7 +3,11 @@ # The main things that need to be set up in the config are # `requirements`, which need to include Hugging Face transformers, # and the secrets. +description: "A tutorial example showing how to deploy private-model on Baseten." environment_variables: {} +model_metadata: + repo_id: "baseten/docs-example-gated-model" + example_model_input: "It is a [MASK] world" model_name: private-model python_version: py39 requirements: diff --git a/09-private-huggingface/doc.yaml b/tutorials/private-huggingface/doc.yaml similarity index 100% rename from 09-private-huggingface/doc.yaml rename to tutorials/private-huggingface/doc.yaml diff --git a/xtts-streaming/model/__init__.py b/tutorials/private-huggingface/model/__init__.py similarity index 100% rename from xtts-streaming/model/__init__.py rename to tutorials/private-huggingface/model/__init__.py diff --git a/09-private-huggingface/model/model.py b/tutorials/private-huggingface/model/model.py similarity index 100% rename from 09-private-huggingface/model/model.py rename to tutorials/private-huggingface/model/model.py diff --git a/tutorials/speech-to-text/README.md b/tutorials/speech-to-text/README.md new file mode 100644 index 000000000..b6c3c8b17 --- /dev/null +++ b/tutorials/speech-to-text/README.md @@ -0,0 +1,31 @@ +# Whisper + +A tutorial example showing how to deploy Whisper on Baseten. + +| Property | Value | +|----------|-------| +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | A10G | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" +}' +``` + +## Configuration highlights + +- System packages: `ffmpeg` diff --git a/05-speech-to-text/config.yaml b/tutorials/speech-to-text/config.yaml similarity index 81% rename from 05-speech-to-text/config.yaml rename to tutorials/speech-to-text/config.yaml index 56ab01839..c202dd6d3 100644 --- a/05-speech-to-text/config.yaml +++ b/tutorials/speech-to-text/config.yaml @@ -1,5 +1,7 @@ +description: "A tutorial example showing how to deploy Whisper on Baseten." environment_variables: {} model_metadata: + repo_id: "openai/whisper-base" example_model_input: {"url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3"} model_name: Whisper python_version: py39 diff --git a/xtts-v2-truss/model/__init__.py b/tutorials/speech-to-text/model/__init__.py similarity index 100% rename from xtts-v2-truss/model/__init__.py rename to tutorials/speech-to-text/model/__init__.py diff --git a/05-speech-to-text/model/model.py b/tutorials/speech-to-text/model/model.py similarity index 100% rename from 05-speech-to-text/model/model.py rename to tutorials/speech-to-text/model/model.py diff --git a/tutorials/system-packages/README.md b/tutorials/system-packages/README.md new file mode 100644 index 000000000..71f969772 --- /dev/null +++ b/tutorials/system-packages/README.md @@ -0,0 +1,32 @@ +# LayoutLM Document QA + +A tutorial example showing how to deploy LayoutLM Document QA on Baseten. + +| Property | Value | +|----------|-------| +| Task | Tutorial | +| Engine | Custom (Truss) | +| GPU | CPU | +| Python | py39 | + +## Deploy + +```sh +truss push +``` + +## Invoke + +```sh +curl -X POST https://model-.api.baseten.co/predict \ + -H "Authorization: Api-Key YOUR_BASETEN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://templates.invoicehome.com/invoice-template-us-neat-750px.png", + "prompt": "What is the invoice number?" +}' +``` + +## Configuration highlights + +- System packages: `tesseract-ocr` diff --git a/10-using-system-packages/config.yaml b/tutorials/system-packages/config.yaml similarity index 90% rename from 10-using-system-packages/config.yaml rename to tutorials/system-packages/config.yaml index a7bc9cfb4..e45fda563 100644 --- a/10-using-system-packages/config.yaml +++ b/tutorials/system-packages/config.yaml @@ -2,9 +2,11 @@ # # The main items that need to be configured in the config.yaml file are requirements # and `system_packages` sections. +description: "A tutorial example showing how to deploy LayoutLM Document QA on Baseten." environment_variables: {} external_package_dirs: [] model_metadata: + repo_id: "impira/layoutlm-document-qa" example_model_input: {"url": "https://templates.invoicehome.com/invoice-template-us-neat-750px.png", "prompt": "What is the invoice number?"} model_name: LayoutLM Document QA python_version: py39 diff --git a/10-using-system-packages/doc.yaml b/tutorials/system-packages/doc.yaml similarity index 100% rename from 10-using-system-packages/doc.yaml rename to tutorials/system-packages/doc.yaml diff --git a/whisper/whisper-truss/data/.gitkeep b/tutorials/system-packages/model/__init__.py similarity index 100% rename from whisper/whisper-truss/data/.gitkeep rename to tutorials/system-packages/model/__init__.py diff --git a/10-using-system-packages/model/model.py b/tutorials/system-packages/model/model.py similarity index 100% rename from 10-using-system-packages/model/model.py rename to tutorials/system-packages/model/model.py diff --git a/ultravox/README.md b/ultravox/README.md deleted file mode 100644 index 773a2a2c1..000000000 --- a/ultravox/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Ultravox vLLM Truss - -This is a [Truss](https://truss.baseten.co/) for Ultravox using the vLLM OpenAI Compatible server. This Truss is designed to provide an efficient and scalable way to serve Ultravox and other models in an OpenAI compatible way using vLLM. - -## OpenAI Bridge Compatibility - -This Truss is compatible with a *custom* version of our [bridge endpoint for OpenAI ChatCompletion users](https://docs.baseten.co/api-reference/openai). This means you can easily integrate this model into your existing applications that use the OpenAI API format. - -``` -client = OpenAI( - api_key=os.environ["BASETEN_API_KEY"], - base_url=f"https://bridge.baseten.co/{model_id}/direct/v1" -) -``` - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp)). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers, and deploy on Baseten. - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples.git -cd ultravox -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `ultravox` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## vLLM OpenAI Compatible Server - -This Truss demonstrates how to start [vLLM's OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html). The Truss is primarily used to start the server and then route requests to it. It currently supports ChatCompletions only. - -### Passing startup arguments to the server - -In the config any key-values under `model_metadata: arguments:` will be passed to the vLLM OpenAI-compatible server at startup. - -### Base Image - -You can use any vLLM compatible base image. - -## API Documentation - -The API follows the OpenAI ChatCompletion format. You can interact with the model using the standard ChatCompletion interface. - -Example usage: - -```python -from openai import OpenAI - -client = OpenAI( - api_key="YOUR-API-KEY", - base_url="https://bridge.baseten.co/MODEL-ID/v1" -) - -response = client.chat.completions.create( - model="fixie-ai/ultravox-v0.2", - messages=[{ - "role": "user", - "content": [ - {"type": "text", "text": "Summarize the following: <|audio|>"}, - {"type": "image_url", "image_url": {"url": f"data:audio/wav;base64,{base64_wav}"}} - ] - }] - stream=True -) - -for chunk in response: - print(chunk.choices[0].delta) -``` - -## Future Improvements - -We are actively working on enhancing this Truss. Some planned improvements include: - -- Adding support for distributed serving with Ray (https://docs.vllm.ai/en/latest/serving/distributed_serving.html) -- Implementing model caching for improved performance - -Stay tuned for updates! - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/whisper/faster-whisper-small/README.md b/whisper/faster-whisper-small/README.md deleted file mode 100644 index 1678d03d3..000000000 --- a/whisper/faster-whisper-small/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Whisper Small - -This is a [Truss](https://truss.baseten.co/) for Whisper Small. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Whisper Small up and running. - -## Faster Whisper Small Implementation - -This implementation of Whisper Small uses [Faster Whisper](https://github.com/SYSTRAN/faster-whisper/tree/master), which is up to 4x faster than openai/whisper for the same accuracy while using less memory. - -## Deployment - -You can deploy this model in just a few clicks from our [model library](), or deploy the Truss, which we'll describe here. - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd whisper/faster-whisper-small -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `whisper/faster-whisper-small` as your working directory, you can deploy the model with: - -```sh -truss push --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -Once your Truss is deployed, you can start using Whisper Small for inference! Navigate to the [Baseten UI](https://app.baseten.co/models) to watch the model build and deploy and invoke it via the REST API for tasks like transcription. - -## Example usage - -Here's a sample script which loads a sample .mp3 file for transcription with Whisper Small: - -```python -import requests -import os - -# Replace the empty string with your model id below -model_id = "" - -# We recommend storing your API key as an environment variable -baseten_api_key = os.environ["BASETEN_API_KEY"] - -data = { - "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" -} - -# Call model endpoint -res = requests.post( - f"https://model-{model_id}.api.baseten.co/production/predict", - headers={"Authorization": f"Api-Key {baseten_api_key}"}, - json=data -) - -# Print the output of the model -print(res.json()) -``` - -Here is the model output: -```json -{'language': 'en', 'language_probability': 0.99072265625, 'duration': 11.52, 'segments': [{'text': ' Four score and seven years ago, our fathers brought forth upon this continent a new nation', 'start': 0.0, 'end': 6.5200000000000005}, {'text': ' conceived in liberty and dedicated to the proposition that all men are created equal.', 'start': 6.5200000000000005, 'end': 11.0}]} -``` diff --git a/whisper/whisper-streaming/README.md b/whisper/whisper-streaming/README.md deleted file mode 100644 index aaa6139cd..000000000 --- a/whisper/whisper-streaming/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Whisper Streaming Truss - -This is a truss for [Whisper Streaming](https://github.com/ufal/whisper_streaming). This truss allows you to stream the whisper transcription chunks as they get generated instead of waiting for the entire transcription to finish. - -This whisper streaming model supports the following whisper models: `tiny.en`, `tiny`, `base.en`, `base`, `small.en`, `small`, `medium.en`, `medium`, `large-v1`, `large-v2`,`large-v3`, `large`. -You can specify the model you want to use inside the `config.yaml` file under the key `whisper_model` in the `model_metadata` section. - -```yaml -model_metadata: - whisper_model: medium -model_name: Whisper Streaming -``` - -## Deploying Whisper Streaming - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `whisper/whisper-streaming` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Whisper API documentation - -The model accepts the following inputs: - -- __audio__(required): The input audio file in the form of a base64 string. -- __chunk_size__(optional): The number of seconds of audio that will get transcribed in each chunk. - - - -## Invoking the model - -```python -import requests -import base64 - -def wav_to_base64(file_path): - with open(file_path, "rb") as wav_file: - binary_data = wav_file.read() - base64_data = base64.b64encode(binary_data) - base64_string = base64_data.decode("utf-8") - return base64_string - -resp = requests.post( - "https://model-.api.baseten.co/development/predict", - headers = {"Authorization": "Api-Key BASETEN-API-KEY"}, - json={"audio": wav_to_base64("/path/to/wav/input_audio_file.wav")}, - stream=True -) - -for content in resp.iter_content(): - print(content.decode("utf-8"), end="", flush=True) -``` diff --git a/whisper/whisper-truss/README.md b/whisper/whisper-truss/README.md deleted file mode 100644 index d5246bc41..000000000 --- a/whisper/whisper-truss/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Whisper Truss - -[Whisper](https://github.com/openai/whisper) is a speech-to-text model by [OpenAI](https://openai.com/blog/whisper/) that transcribes audio in dozens of languages with remarkable accuracy. It is open-source under the [MIT license](https://github.com/openai/whisper/blob/main/LICENSE) and hosted on Baseten as a pre-trained model. Read the [Whisper model card](https://github.com/openai/whisper/blob/main/model-card.md) for more details. - -Whisper's leap in transcription quality unlocks tons of compelling use cases, including: - -- Moderating audio content -- Auditing call center logs -- Automatically generating video subtitles -- Improving podcast SEO with transcripts - -## Deploying Whisper - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `whisper-truss` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking Whisper - -Once the model is deployed, you can invoke it with: - -```sh -truss predict -d '{"url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3"}' -``` - -You can also invoke your Whisper deployment via its REST API endpoint: - -```bash -curl -X POST "https://app.baseten.co/models/{MODEL_ID}/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{"url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3"}' -``` - -### Whisper API documentation - -#### Input - -This deployment of Whisper takes input as a JSON dictionary with the key `url` corresponding to a string of a URL pointing at an MP3 file. For example: - -```json -{ - "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" -} -``` - -#### Output - -The model returns a fairly lengthy dictionary. For most uses, you'll be interested in the key `language` which specifies the detected language of the audio and `text` which contains the full transcription. - -```json -{ - "language": "english", - "segments": [ - { - "start": 0, - "end": 6.5200000000000005, - "text": " Four score and seven years ago, our fathers brought forth upon this continent a new nation" - }, - { - "start": 6.52, - "end": 21.6, - "text": " conceived in liberty and dedicated to the proposition that all men are created equal." - } - ], - "text": " Four score and seven years ago, our fathers brought forth upon this continent..." -} -``` diff --git a/whisper/whisper-v3-truss-base64/README.md b/whisper/whisper-v3-truss-base64/README.md deleted file mode 100644 index ddd2b34c2..000000000 --- a/whisper/whisper-v3-truss-base64/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Whisper v3 Truss - -[Whisper](https://github.com/openai/whisper) is a open-source speech-to-text model by [OpenAI](https://openai.com/blog/whisper/) that transcribes audio in dozens of languages with remarkable accuracy. - -Whisper v3 has the same architecture as the previous model but has made the following improvements: -- The large-v3 model is trained on 1 million hours of weakly labeled audio and 4 million hours of pseudolabeled audio collected using large-v2 -- This version has a 10-20% lower rate of error compared to the previous version when benchmarked on `Common Voice 15` and `Fleurs` dataset - - -## Deploying Whisper - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `whisper-v3-truss` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking Whisper - -Once the model is deployed, you can invoke it with: - -```python -import requests -import base64 - - -audio_base64 = base64.b64encode(open("Gettysburg.mp3", "rb").read()).decode('utf-8') - -resp = requests.post( - "https://model-{MODEL_ID}.api.baseten.co/development/predict", - headers={"Authorization": "Api-Key $BASETEN_API_KEY"}, - json={'audio': audio_base64}, -) - -print(resp.content) -``` - -### Whisper API documentation - -#### Input - -This deployment of Whisper takes input as a JSON dictionary with the key `audio` corresponding to a base64 encoded audio file. Here is an example input JSON: - -```json -{ - "audio": "YmG4DdeoS0HEV..." -} -``` - -#### Output - -The model returns a fairly lengthy dictionary. For most uses, you'll be interested in the key `language` which specifies the detected language of the audio and `text` which contains the full transcription. - -```json -{ - "language": "english", - "segments": [ - { - "start": 0, - "end": 11.52, - "text": "Four score and seven years ago our fathers brought forth upon this continent a new nation conceived in liberty and dedicated to the proposition that all men are created equal." - } - ], - "text": "Four score and seven years ago our fathers brought forth upon this continent a new nation conceived in liberty and dedicated to the proposition that all men are created equal." -} -``` diff --git a/whisper/whisper-v3-truss/README.md b/whisper/whisper-v3-truss/README.md deleted file mode 100644 index 3894fb926..000000000 --- a/whisper/whisper-v3-truss/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# Whisper v3 Truss - -[Whisper](https://github.com/openai/whisper) is a open-source speech-to-text model by [OpenAI](https://openai.com/blog/whisper/) that transcribes audio in dozens of languages with remarkable accuracy. - -Whisper v3 has the same architecture as the previous model but has made the following improvements: -- The large-v3 model is trained on 1 million hours of weakly labeled audio and 4 million hours of pseudolabeled audio collected using large-v2 -- This version has a 10-20% lower rate of error compared to the previous version when benchmarked on `Common Voice 15` and `Fleurs` dataset - - -## Deploying Whisper - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `whisper-v3-truss` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking Whisper - -Once the model is deployed, you can invoke it with: - -```sh -truss predict -d '{"url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3"}' -``` - -You can also invoke your Whisper deployment via its REST API endpoint: - -```bash -curl -X POST "https://model-{MODEL_ID}.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{"url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3"}' -``` - -### Whisper API documentation - -#### Input - -This deployment of Whisper takes input as a JSON dictionary with the key `url` corresponding to a string of a URL pointing at an MP3 file. For example: - -```json -{ - "url": "https://cdn.baseten.co/docs/production/Gettysburg.mp3" -} -``` - -#### Output - -The model returns a fairly lengthy dictionary. For most uses, you'll be interested in the key `language` which specifies the detected language of the audio and `text` which contains the full transcription. - -```json -{ - "language": "english", - "segments": [ - { - "start": 0, - "end": 11.52, - "text": "Four score and seven years ago our fathers brought forth upon this continent a new nation conceived in liberty and dedicated to the proposition that all men are created equal." - } - ], - "text": "Four score and seven years ago our fathers brought forth upon this continent a new nation conceived in liberty and dedicated to the proposition that all men are created equal." -} -``` diff --git a/whisper/whisperx-truss/README.md b/whisper/whisperx-truss/README.md deleted file mode 100644 index 1582c867f..000000000 --- a/whisper/whisperx-truss/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# WhisperX Truss - -[WhisperX](https://github.com/m-bain/whisperX) is a model built on top of Whisper that provides fast speech recognition with word-level timestamps. - -## Getting Access To The Model -The base WhisperX model does not need any special requirements, but if you want to enable speaker diarization there are a couple of models that you will need to get permission from. -1. Go to the [speaker-diarization model](https://huggingface.co/pyannote/speaker-diarization-3.0) and fill out the required info to gain access to the model. - -2. Go to the [segmentation model](https://huggingface.co/pyannote/segmentation-3.0) and go through the same process. - -Once you have access to both of those models, make sure you have your hugging face access token on hand as you will need it to run this truss. - -1. Create a [HuggingFace access token](https://huggingface.co/settings/tokens) -2. Set it as a [secret in your Baseten account](https://app.baseten.co/settings/secrets) with the name `hf_access_token` - -## Deploying WhisperX - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -Next, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd whisperx-truss -``` - -With `whisperx-truss` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Hardware notes - -This whisperX model comes in various sizes: ["small", "medium", "large-v2"]. -The large-v2 model can be easily run on the T4 GPU. - -## API route: `predict` - -The `predict` route is the primary method used for audio transcription. - -- __audio_file__: An MP3 audio file. This file must be accessible over the internet as files from local storage are not accessible. - -## Invoking WhisperX - -Once the model is deployed, you can invoke it with: - -```sh -truss predict -d '{"audio_file": "https://cdn.baseten.co/docs/production/Gettysburg.mp3"}' -``` - -You can also invoke your Whisper deployment via its REST API endpoint: - -```bash -curl -X POST "https://app.baseten.co/models/{MODEL_ID}/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{"audio_file": "https://cdn.baseten.co/docs/production/Gettysburg.mp3"}' -``` - -## Output - -The model returns a dictionary which contains the start and end timestamps along with the transcript of what is said in between each timestamp. - -```json -{ - "model_output": - [ - { - "end": 10.742, - "start": 0.765, - "text": "Four score and seven years ago, our fathers brought forth upon this continent, a new nation conceived in liberty and dedicated to the proposition that all men are created equal.", - "speaker": "SPEAKER_00" - } - ] -} -``` diff --git a/xtts-streaming/README.md b/xtts-streaming/README.md deleted file mode 100644 index 795a7d817..000000000 --- a/xtts-streaming/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# XTTS Streaming - -This repository packages [TTS](https://github.com/coqui-ai/TTS) as a [Truss](https://truss.baseten.co/) but with streaming. - -TTS is a generative audio model for text-to-speech generation. This model takes in text and a speaker's voice as input and converts the text to speech in the voice of the speaker. - -## Deploying XTTS - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd xtts-streaming -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `xtts-v2-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking the model - -Here are the following inputs for the model: -1. `text`: The text that needs to be converted into speech -2. `language`: Language for the text -3. `chunk_size`: Integer size of each chunk being streamed - -Here are two examples of streaming the audio. This first example write all of the streamed chunks to an audio file. - -```python -import wave -import requests - -channels = 1 # mono=1, stereo=2 -sampwidth = 2 # Sample width in bytes, typical values: 2 for 16-bit audio, 1 for 8-bit audio -framerate = 24000 # Sampling rate, in samples per second (Hz) - - -resp = requests.post( - "https://model-.api.baseten.co/development/predict", - headers={"Authorization": "Api-Key BASETEN-API-KEY"}, - json={"text": "Kurt watched the incoming Pelicans. The blocky jet-powered craft were so distant they were only specks against the setting sun. He hit the magnification on his faceplate and saw lines of fire tracing their reentry vectors. They would touch down in three minutes."}, - stream=True -) - -with wave.open("dat2-wav.wav", 'wb') as wav_file: - wav_file.setnchannels(channels) - wav_file.setsampwidth(sampwidth) - wav_file.setframerate(framerate) - - # Iterate through streamed content and write audio chunks directly - for chunk in resp.iter_content(chunk_size=None): # Use server's chunk size - if chunk: - wav_file.writeframes(chunk) -``` - -If you want to stream the audio directly as it gets generated here is another option: - -```python -import pyaudio - -FORMAT = pyaudio.paInt16 # Audio format (e.g., 16-bit PCM) -CHANNELS = 1 # Number of audio channels -RATE = 24000 # Sample rate - -# Initialize PyAudio -p = pyaudio.PyAudio() - -# Open a stream for audio playback -stream = p.open(format=p.get_format_from_width(2), channels=CHANNELS, rate=RATE, output=True) - -# Make a streaming HTTP request to the server -original_text = "Kurt watched the incoming Pelicans. The blocky jet-powered craft were so distant they were only specks against the setting sun. He hit the magnification on his faceplate and saw lines of fire tracing their reentry vectors. They would touch down in three minutes." - - -resp = requests.post( - "https://model-.api.baseten.co/development/predict", - headers={"Authorization": "Api-Key BASETEN-API-KEY"}, - json={"text": "Kurt watched the incoming Pelicans. The blocky jet-powered craft were so distant they were only specks against the setting sun. He hit the magnification on his faceplate and saw lines of fire tracing their reentry vectors. They would touch down in three minutes."}, - stream=True -) - -# Create a buffer to hold multiple chunks -buffer = b'' -buffer_size_threshold = 2**20 - -# Stream and play the audio data as it's received -for chunk in resp.iter_content(chunk_size=4096): - if chunk: - buffer += chunk - if len(buffer) >= buffer_size_threshold: - print(f"Writing buffer of size: {len(buffer)}") - stream.write(buffer) - buffer = b'' # Clear the buffer - # stream.write(chunk) - -if buffer: - print(f"Writing final buffer of size: {len(buffer)}") - stream.write(buffer) - -# Close and terminate the stream and PyAudio -stream.stop_stream() -stream.close() -p.terminate() -``` diff --git a/xtts-v2-truss/README.md b/xtts-v2-truss/README.md deleted file mode 100644 index dae2c255e..000000000 --- a/xtts-v2-truss/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# XTTS V2 Truss - -This repository packages [TTS](https://github.com/coqui-ai/TTS) as a [Truss](https://truss.baseten.co/). - -TTS is a generative audio model for text-to-speech generation. This model takes in text and a speaker's voice as input and converts the text to speech in the voice of the speaker. - -## Deploying XTTS - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd xtts-v2-truss -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `xtts-v2-truss` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Invoking the model - -Here are the following inputs for the model: -1. `text`: The text that needs to be converted into speech -2. `speaker_voice`: A short audio clip of a voice in the format of a base64 string -3. `language`: Abbreviation of the supported languages for TTS - -Here is an example of how to invoke this model: - -```python - -import base64 -import sys - -def wav_to_base64(file_path): - with open(file_path, "rb") as wav_file: - binary_data = wav_file.read() - base64_data = base64.b64encode(binary_data) - base64_string = base64_data.decode("utf-8") - return base64_string - -def base64_to_wav(base64_string, output_file_path): - binary_data = base64.b64decode(base64_string) - with open(output_file_path, "wb") as wav_file: - wav_file.write(binary_data) - -voice = wav_to_base64("/path/to/wav/file/samuel_jackson_voice.wav") -text = "Listen up, people. Life's a wild ride, and sometimes you gotta grab it by the horns and steer it where you want to go. You can't just sit around waiting for things to happen – you gotta make 'em happen. Yeah, it's gonna get tough, but that's when you dig deep, find that inner badass, and come out swinging. Remember, success ain't handed to you on a silver platter; you gotta snatch it like it owes you money. So, lace up your boots, square those shoulders, and let the world know that you're here to play, and you're playing for keeps" -data = {"text": text, "speaker_voice": voice, "language": "en"} -res = requests.post("https://model-.api.baseten.co/development/predict", headers=headers, json=data) -res = res.json() -output = base64_to_wav(res.get('output'), "test_output.wav") -``` - -The output of the model is a base64 string as well, so you can convert it to a wav file using the `base64_to_wav` function. - -Here is the input file for Samuel Jackson's voice: -![speaker voice](https://github.com/htrivedi99/truss-examples/assets/15642666/6ae79c53-f63a-4e0a-b6fc-d397fee6162e) - -Here is the output file for the text to speech: -![output](https://github.com/htrivedi99/truss-examples/assets/15642666/2a2f9a80-860d-4ef1-a530-7892a8bd874e) - -In this README, the audio files are in the MP4 format because github does not allow the WAV format. However, for the truss itself using the WAV format is recommended. diff --git a/z-ai/glm-4-5-air-fp8/README.md b/z-ai/glm-4-5-air-fp8/README.md deleted file mode 100644 index cd46514a1..000000000 --- a/z-ai/glm-4-5-air-fp8/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# GLM 4.5 Air FP8 (SGLang) - -This example shows how to deploy [GLM 4.5 Air in FP8](https://huggingface.co/zai-org/GLM-4.5-Air-FP8) using SGLang on Baseten. This model requires 4xH100 to access the full context window. - -## Deploying the model - -This model can be deployed to Baseten using Truss: - -``` -pip install --upgrade truss -truss push --publish z-ai/glm-4-5-air-fp8 -``` - -## Calling the model - -This model is OpenAI compatible and can be called using the OpenAI client. - -```python -import os -from openai import OpenAI - -# https://model-XXXXXXX.api.baseten.co/environments/production/sync/v1 -model_url = "" - -client = OpenAI( - base_url=model_url, - api_key=os.environ.get("BASETEN_API_KEY"), -) - -stream = client.chat.completions.create( - model="baseten", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Write FizzBuzz."} - ], - stream=True, -) - -for chunk in stream: - if chunk.choices[0].delta.content is not None: - print(chunk.choices[0].delta.content, end="") -``` diff --git a/z-ai/glm-4-5-fp8/README.md b/z-ai/glm-4-5-fp8/README.md deleted file mode 100644 index 1b3e3a4ab..000000000 --- a/z-ai/glm-4-5-fp8/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# GLM 4.5 FP8 (SGLang) - -This example shows how to deploy [GLM 4.5 in FP8](https://huggingface.co/zai-org/GLM-4.5-FP8) using SGLang on Baseten. This model requires 8xH100 to run and H200 or B200 to access the full context window. - -## Deploying the model - -This model can be deployed to Baseten using Truss: - -``` -pip install --upgrade truss -truss push --publish z-ai/glm-4-5-fp8 -``` - -## Calling the model - -This model is OpenAI compatible and can be called using the OpenAI client. - -```python -import os -from openai import OpenAI - -# https://model-XXXXXXX.api.baseten.co/environments/production/sync/v1 -model_url = "" - -client = OpenAI( - base_url=model_url, - api_key=os.environ.get("BASETEN_API_KEY"), -) - -stream = client.chat.completions.create( - model="baseten", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Write FizzBuzz."} - ], - stream=True, -) - -for chunk in stream: - if chunk.choices[0].delta.content is not None: - print(chunk.choices[0].delta.content, end="") -``` From 0a3be9a16426927367b0bc5381f005909d7a48ad Mon Sep 17 00:00:00 2001 From: Pat Date: Thu, 12 Feb 2026 13:30:10 -0600 Subject: [PATCH 2/4] Fix ruff lint and formatting errors for CI Remove unused import and fix f-string without placeholders in generate_readmes.py. Apply ruff formatting to 13 files. --- _internal/bin/generate_readmes.py | 65 +++++-- _internal/bin/test_all.py | 167 +++++++++++++++--- _internal/bin/validate_all.py | 8 +- _internal/templating/generate_templates.py | 8 +- .../Sana/diffusion/model/dpm_solver.py | 8 +- .../Sana/diffusion/model/edm_sample.py | 21 +-- .../Sana/diffusion/model/sa_solver.py | 6 +- .../Sana/diffusion/model/dpm_solver.py | 8 +- .../Sana/diffusion/model/edm_sample.py | 21 +-- .../Sana/diffusion/model/sa_solver.py | 6 +- .../src/mvdream/ldm/models/diffusion/ddim.py | 4 +- .../src/mvdream/ldm/modules/attention.py | 12 +- .../ldm/modules/diffusionmodules/model.py | 12 +- 13 files changed, 248 insertions(+), 98 deletions(-) diff --git a/_internal/bin/generate_readmes.py b/_internal/bin/generate_readmes.py index e866c9567..14dc36d9d 100644 --- a/_internal/bin/generate_readmes.py +++ b/_internal/bin/generate_readmes.py @@ -6,7 +6,6 @@ import json import os import re -import sys from pathlib import Path import yaml @@ -19,6 +18,7 @@ # Metadata extraction helpers # --------------------------------------------------------------------------- + def extract_hf_id(config: dict) -> str | None: # 1. model_metadata.repo_id repo_id = config.get("model_metadata", {}).get("repo_id") @@ -52,7 +52,11 @@ def extract_hf_id(config: dict) -> str | None: def detect_engine(config: dict) -> str: - base_image = str(config.get("base_image", {}).get("image", "") if isinstance(config.get("base_image"), dict) else "") + base_image = str( + config.get("base_image", {}).get("image", "") + if isinstance(config.get("base_image"), dict) + else "" + ) start_cmd = config.get("docker_server", {}).get("start_command", "") requirements = [str(r) for r in config.get("requirements", [])] req_str = " ".join(requirements) @@ -67,7 +71,10 @@ def detect_engine(config: dict) -> str: return "vLLM" if "sglang" in start_cmd.lower() or "sglang" in req_str.lower(): return "SGLang" - if "text-embeddings" in base_image.lower() or "text-embeddings-router" in start_cmd.lower(): + if ( + "text-embeddings" in base_image.lower() + or "text-embeddings-router" in start_cmd.lower() + ): return "TEI (HuggingFace)" if config.get("docker_server"): return "Docker Server" @@ -103,13 +110,25 @@ def infer_task_type(config: dict, category: str, dir_name: str) -> str: # Check if it's a reranker if "rerank" in dir_name.lower(): return "Reranking" - if "classification" in dir_name.lower() or "reward" in dir_name.lower() or "ner" in dir_name.lower(): + if ( + "classification" in dir_name.lower() + or "reward" in dir_name.lower() + or "ner" in dir_name.lower() + ): return "Classification" return "Embeddings" if "image" in cat_l: return "Image generation" if "audio" in cat_l: - if "tts" in dir_name.lower() or "voice" in dir_name.lower() or "speech" in dir_name.lower() or "kokoro" in dir_name.lower() or "chatterbox" in dir_name.lower() or "metavoice" in dir_name.lower() or "sesame" in dir_name.lower(): + if ( + "tts" in dir_name.lower() + or "voice" in dir_name.lower() + or "speech" in dir_name.lower() + or "kokoro" in dir_name.lower() + or "chatterbox" in dir_name.lower() + or "metavoice" in dir_name.lower() + or "sesame" in dir_name.lower() + ): return "Text-to-speech" if "whisper" in dir_name.lower() or "transcri" in dir_name.lower(): return "Speech-to-text" @@ -211,6 +230,7 @@ def get_subcategory(example_dir: Path) -> str: # Config highlights # --------------------------------------------------------------------------- + def build_config_highlights(config: dict, engine: str) -> list[str]: highlights = [] @@ -263,7 +283,9 @@ def build_config_highlights(config: dict, engine: str) -> list[str]: if caches and isinstance(caches, list): for c in caches: if c.get("use_volume"): - highlights.append("Model cache: **volume-mounted** for fast cold starts") + highlights.append( + "Model cache: **volume-mounted** for fast cold starts" + ) break # Concurrency @@ -284,9 +306,13 @@ def build_config_highlights(config: dict, engine: str) -> list[str]: # Environment variables (non-secret) envs = config.get("environment_variables", {}) if isinstance(envs, dict): - notable = {k: v for k, v in envs.items() if k != "hf_access_token" and v is not None} + notable = { + k: v for k, v in envs.items() if k != "hf_access_token" and v is not None + } if notable: - highlights.append(f"Environment variables: {', '.join(f'`{k}`' for k in notable)}") + highlights.append( + f"Environment variables: {', '.join(f'`{k}`' for k in notable)}" + ) if not highlights: highlights.append(f"Engine: **{engine}**") @@ -298,6 +324,7 @@ def build_config_highlights(config: dict, engine: str) -> list[str]: # Invoke section # --------------------------------------------------------------------------- + def build_invoke_section( config: dict, engine: str, @@ -440,7 +467,15 @@ def build_invoke_section( # Description generation # --------------------------------------------------------------------------- -def build_description(model_name: str, config: dict, category: str, engine: str, hf_id: str | None, task: str) -> str: + +def build_description( + model_name: str, + config: dict, + category: str, + engine: str, + hf_id: str | None, + task: str, +) -> str: desc = config.get("description") if desc: return desc @@ -478,6 +513,7 @@ def build_description(model_name: str, config: dict, category: str, engine: str, # README rendering # --------------------------------------------------------------------------- + def render_readme( model_name: str, description: str, @@ -517,7 +553,9 @@ def render_readme( # Deploy lines.append("## Deploy\n") if hf_token: - lines.append("> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying.\n") + lines.append( + "> **Note:** This model requires a HuggingFace access token. Set `hf_access_token` in your Baseten secrets before deploying.\n" + ) lines.append("```sh\ntruss push\n```\n") # Invoke @@ -538,6 +576,7 @@ def render_readme( # Main # --------------------------------------------------------------------------- + def find_example_dirs() -> list[Path]: """Find all directories containing config.yaml, excluding archive/internal.""" examples = [] @@ -558,7 +597,9 @@ def process_example(example_dir: Path) -> dict: dir_name = example_dir.name category = get_category(example_dir) - model_name = config.get("model_name") or dir_name.replace("-", " ").replace("_", " ").title() + model_name = ( + config.get("model_name") or dir_name.replace("-", " ").replace("_", " ").title() + ) hf_id = extract_hf_id(config) engine = detect_engine(config) task = infer_task_type(config, category, dir_name) @@ -626,7 +667,7 @@ def main(): if not result["hf_id"]: missing_hf.append(str(rel)) - print(f"\n--- Summary ---") + print("\n--- Summary ---") print(f"Generated: {generated}/{len(examples)}") print(f"Missing HuggingFace ID: {len(missing_hf)}") if missing_hf: diff --git a/_internal/bin/test_all.py b/_internal/bin/test_all.py index 6bb0e66fe..4789acc15 100644 --- a/_internal/bin/test_all.py +++ b/_internal/bin/test_all.py @@ -27,7 +27,9 @@ try: import truss except ImportError: - print("ERROR: truss not installed. Run: uv pip install -e ../truss", file=sys.stderr) + print( + "ERROR: truss not installed. Run: uv pip install -e ../truss", file=sys.stderr + ) sys.exit(1) REPO_ROOT = Path(__file__).resolve().parent.parent.parent @@ -38,12 +40,30 @@ CATEGORY_FILTER = sys.argv[i + 1] SKIP_DIRS = { - ".git", ".github", "__pycache__", "node_modules", ".venv", "venv", - "templating", "assets", "bin", "dockerfiles", "packages", "sample_images", + ".git", + ".github", + "__pycache__", + "node_modules", + ".venv", + "venv", + "templating", + "assets", + "bin", + "dockerfiles", + "packages", + "sample_images", "examples", # skip nested examples dirs for dir naming checks } -CUSTOMER_CATEGORIES = ["tutorials", "llm", "embeddings", "image", "audio", "optimized", "infrastructure"] +CUSTOMER_CATEGORIES = [ + "tutorials", + "llm", + "embeddings", + "image", + "audio", + "optimized", + "infrastructure", +] # Counters passed = 0 @@ -77,7 +97,12 @@ def find_all_configs(root: Path) -> list[Path]: """Find all directories containing a config.yaml.""" configs = [] for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [d for d in dirnames if d not in {".git", ".github", "__pycache__", "node_modules", ".venv", "venv"}] + dirnames[:] = [ + d + for d in dirnames + if d + not in {".git", ".github", "__pycache__", "node_modules", ".venv", "venv"} + ] if "config.yaml" in filenames: rel = Path(dirpath).relative_to(root) # Skip _archive @@ -91,6 +116,7 @@ def find_all_configs(root: Path) -> list[Path]: # ─── Test 1: Config validation ─────────────────────────────────────────────── + def test_config_validation(): print("\n== Test 1: Config Validation (YAML + truss.load) ==") configs = find_all_configs(REPO_ROOT) @@ -124,6 +150,7 @@ def test_config_validation(): # ─── Test 2: README existence ──────────────────────────────────────────────── + def test_readme_existence(): print("\n== Test 2: README Existence ==") configs = find_all_configs(REPO_ROOT) @@ -131,7 +158,11 @@ def test_readme_existence(): for config_dir in configs: rel = config_dir.relative_to(REPO_ROOT) - if str(rel).startswith("_internal") or "/_archive/" in str(rel) or str(rel).startswith("_archive"): + if ( + str(rel).startswith("_internal") + or "/_archive/" in str(rel) + or str(rel).startswith("_archive") + ): continue # Skip _archive at any level if "_archive" in rel.parts: @@ -156,6 +187,7 @@ def test_readme_existence(): # ─── Test 3: README ↔ Config consistency ───────────────────────────────────── + def test_readme_config_consistency(): print("\n== Test 3: README ↔ Config Consistency ==") configs = find_all_configs(REPO_ROOT) @@ -183,10 +215,15 @@ def test_readme_config_consistency(): # Check: if config has hf_access_token secret, README should mention it secrets = raw.get("secrets", {}) if secrets and isinstance(secrets, dict) and "hf_access_token" in secrets: - if "hf_access_token" in readme_text or "HuggingFace access token" in readme_text: + if ( + "hf_access_token" in readme_text + or "HuggingFace access token" in readme_text + ): log_pass(f"{rel}: README mentions HF token (config requires it)") else: - log_warn(f"{rel}: config has hf_access_token but README doesn't mention it") + log_warn( + f"{rel}: config has hf_access_token but README doesn't mention it" + ) # Check: endpoint consistency docker_server = raw.get("docker_server", {}) @@ -197,7 +234,9 @@ def test_readme_config_consistency(): if predict_endpoint in readme_text: log_pass(f"{rel}: README uses correct endpoint {predict_endpoint}") elif "/predict" in readme_text and predict_endpoint not in readme_text: - log_warn(f"{rel}: README uses /predict but config has {predict_endpoint}") + log_warn( + f"{rel}: README uses /predict but config has {predict_endpoint}" + ) # Check: OpenAI-compatible tag should have /v1/chat/completions in README tags = [] @@ -208,11 +247,14 @@ def test_readme_config_consistency(): if "/v1/chat/completions" in readme_text or "OpenAI" in readme_text: log_pass(f"{rel}: OpenAI-compatible model has correct invoke style") else: - log_warn(f"{rel}: tagged openai-compatible but README lacks /v1/chat/completions") + log_warn( + f"{rel}: tagged openai-compatible but README lacks /v1/chat/completions" + ) # ─── Test 4: Directory naming conventions ──────────────────────────────────── + def test_directory_naming(): print("\n== Test 4: Directory Naming Conventions ==") @@ -221,7 +263,9 @@ def test_directory_naming(): if not cat_dir.exists(): continue for dirpath, dirnames, filenames in os.walk(cat_dir): - dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".")] + dirnames[:] = [ + d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".") + ] rel = Path(dirpath).relative_to(REPO_ROOT) # If this directory contains config.yaml, it's an example root. @@ -236,19 +280,26 @@ def test_directory_naming(): # Check underscores (allow go_emotions which is upstream model name) if "_" in d and "go_emotions" not in d: - log_fail(f"{rel}/{d}: directory name contains underscore (use hyphens)") + log_fail( + f"{rel}/{d}: directory name contains underscore (use hyphens)" + ) # Check for "truss" in name - if "truss" in d.lower() and d != "truss": # allow ngram-speculator/truss + if ( + "truss" in d.lower() and d != "truss" + ): # allow ngram-speculator/truss log_warn(f"{rel}/{d}: directory name contains 'truss'") # Check PascalCase (first char uppercase suggests PascalCase) if d[0].isupper() and "-" in d: - log_warn(f"{rel}/{d}: directory name uses PascalCase (prefer lowercase)") + log_warn( + f"{rel}/{d}: directory name uses PascalCase (prefer lowercase)" + ) # ─── Test 5: Link/path validation in READMEs ──────────────────────────────── + def test_readme_links(): print("\n== Test 5: README Link Validation ==") @@ -257,7 +308,7 @@ def test_readme_links(): if root_readme.exists(): text = root_readme.read_text() # Find all relative links like [text](path/) - links = re.findall(r'\[.*?\]\(([^)]+)\)', text) + links = re.findall(r"\[.*?\]\(([^)]+)\)", text) for link in links: if link.startswith("http") or link.startswith("#"): continue @@ -275,7 +326,7 @@ def test_readme_links(): if not cat_readme.exists(): continue text = cat_readme.read_text() - links = re.findall(r'\[.*?\]\(([^)]+)\)', text) + links = re.findall(r"\[.*?\]\(([^)]+)\)", text) for link in links: if link.startswith("http") or link.startswith("#"): continue @@ -291,23 +342,35 @@ def test_readme_links(): if contrib.exists(): text = contrib.read_text() # Find paths in the table (backtick-wrapped) - paths = re.findall(r'`([a-z][\w/-]+)`', text) + paths = re.findall(r"`([a-z][\w/-]+)`", text) for p in paths: - if "/" in p and not p.endswith(".py") and not p.endswith(".yaml") and not p.endswith(".yml"): + if ( + "/" in p + and not p.endswith(".py") + and not p.endswith(".yaml") + and not p.endswith(".yml") + ): target = REPO_ROOT / p if target.exists(): log_pass(f"CONTRIBUTING.md: path {p} exists") - elif not any(p.endswith(ext) for ext in [".yaml", ".py", ".md", ".json"]): + elif not any( + p.endswith(ext) for ext in [".yaml", ".py", ".md", ".json"] + ): # Check if it exists as a subdir within any category - found = any((REPO_ROOT / cat / p).exists() for cat in CUSTOMER_CATEGORIES) + found = any( + (REPO_ROOT / cat / p).exists() for cat in CUSTOMER_CATEGORIES + ) if found: - log_pass(f"CONTRIBUTING.md: path {p} exists (relative to category)") + log_pass( + f"CONTRIBUTING.md: path {p} exists (relative to category)" + ) else: log_warn(f"CONTRIBUTING.md: path {p} may not exist") # ─── Test 6: example_model_input validation ────────────────────────────────── + def test_example_model_input(): print("\n== Test 6: example_model_input Validation ==") configs = find_all_configs(REPO_ROOT) @@ -342,22 +405,45 @@ def test_example_model_input(): if rel_str.startswith("llm/"): # LLMs: prompt, messages, query, text, message, image_url, queries all valid if isinstance(example_input, dict): - valid_keys = {"prompt", "messages", "query", "text", "message", "image_url", "queries", "model"} + valid_keys = { + "prompt", + "messages", + "query", + "text", + "message", + "image_url", + "queries", + "model", + } if valid_keys & set(example_input.keys()): log_pass(f"{rel}: example_model_input has valid LLM format") else: - log_warn(f"{rel}: LLM example_model_input missing prompt/messages/query") + log_warn( + f"{rel}: LLM example_model_input missing prompt/messages/query" + ) elif isinstance(example_input, str): log_pass(f"{rel}: example_model_input is a string (direct input)") elif rel_str.startswith("embeddings/"): # Embeddings: input (standard), query/texts (rerankers), text/inputs (classifiers/NER), url (CLIP) if isinstance(example_input, dict): - valid_keys = {"input", "inputs", "encoding_format", "query", "texts", "text", "model", "sentences", "url"} + valid_keys = { + "input", + "inputs", + "encoding_format", + "query", + "texts", + "text", + "model", + "sentences", + "url", + } if valid_keys & set(example_input.keys()): log_pass(f"{rel}: example_model_input has valid embedding format") else: - log_warn(f"{rel}: embedding example_model_input may have wrong format") + log_warn( + f"{rel}: embedding example_model_input may have wrong format" + ) elif isinstance(example_input, str): log_pass(f"{rel}: example_model_input is a string (direct input)") else: @@ -366,7 +452,19 @@ def test_example_model_input(): elif rel_str.startswith("image/"): # Image: prompt, image, instances, workflow_values (comfyui), input_image, reference_image, image_url if isinstance(example_input, dict): - valid_keys = {"prompt", "image", "instances", "workflow", "workflow_values", "url", "input_image", "reference_image", "bbox", "image_url", "text"} + valid_keys = { + "prompt", + "image", + "instances", + "workflow", + "workflow_values", + "url", + "input_image", + "reference_image", + "bbox", + "image_url", + "text", + } if valid_keys & set(example_input.keys()): log_pass(f"{rel}: example_model_input has valid image format") else: @@ -389,6 +487,7 @@ def test_example_model_input(): # ─── Test 7: Requirements pinning ──────────────────────────────────────────── + def test_requirements_pinning(): print("\n== Test 7: Requirements Pinning ==") configs = find_all_configs(REPO_ROOT) @@ -429,7 +528,12 @@ def test_requirements_pinning(): unpinned.append(req) continue # Check for version pin - if "==" not in req and ">=" not in req and "<=" not in req and "~=" not in req: + if ( + "==" not in req + and ">=" not in req + and "<=" not in req + and "~=" not in req + ): unpinned.append(req) if unpinned: @@ -445,6 +549,7 @@ def test_requirements_pinning(): # ─── Test 8: CI path validation ────────────────────────────────────────────── + def test_ci_paths(): print("\n== Test 8: CI Path Validation ==") ci_path = REPO_ROOT / "ci.yaml" @@ -466,13 +571,16 @@ def test_ci_paths(): truss.load(str(full_path)) log_pass(f"ci.yaml: {test_path} exists and loads") except Exception as e: - log_fail(f"ci.yaml: {test_path} exists but truss.load() fails: {str(e)[:100]}") + log_fail( + f"ci.yaml: {test_path} exists but truss.load() fails: {str(e)[:100]}" + ) else: log_fail(f"ci.yaml: {test_path} does not exist") # ─── Test 9: TRT-LLM openai-compatible tag ────────────────────────────────── + def test_trt_llm_tags(): print("\n== Test 9: TRT-LLM openai-compatible Tag ==") configs = find_all_configs(REPO_ROOT) @@ -513,6 +621,7 @@ def test_trt_llm_tags(): # ─── Test 10: model.py syntax validation ──────────────────────────────────── + def test_model_py_syntax(): print("\n== Test 10: model.py Syntax Validation ==") import ast @@ -536,6 +645,7 @@ def test_model_py_syntax(): # ─── Test 10: CI completeness ─────────────────────────────────────────────── + def test_ci_completeness(): print("\n== Test 11: CI Completeness (ci.yaml covers all examples) ==") ci_path = REPO_ROOT / "ci.yaml" @@ -569,6 +679,7 @@ def test_ci_completeness(): # ─── Main ──────────────────────────────────────────────────────────────────── + def main(): print("=" * 60) print(" Truss Examples - Comprehensive Test Suite") diff --git a/_internal/bin/validate_all.py b/_internal/bin/validate_all.py index e0e6036f7..d05f8af76 100644 --- a/_internal/bin/validate_all.py +++ b/_internal/bin/validate_all.py @@ -21,7 +21,9 @@ import truss from truss.base.truss_config import TrussConfig except ImportError: - print("ERROR: truss not installed. Run: uv pip install -e ../truss", file=sys.stderr) + print( + "ERROR: truss not installed. Run: uv pip install -e ../truss", file=sys.stderr + ) sys.exit(1) REPO_ROOT = Path(__file__).resolve().parent.parent.parent @@ -337,7 +339,9 @@ def main(): "warnings": r.warnings, } ) - report_path.write_text(json.dumps({"summary": status_counts, "results": data}, indent=2)) + report_path.write_text( + json.dumps({"summary": status_counts, "results": data}, indent=2) + ) print(f"\nReport also saved to {report_path}", file=sys.stderr) diff --git a/_internal/templating/generate_templates.py b/_internal/templating/generate_templates.py index 093c1819b..625f2f99f 100644 --- a/_internal/templating/generate_templates.py +++ b/_internal/templating/generate_templates.py @@ -1168,7 +1168,7 @@ def generate_deployment(dp: Deployment): raw_folder = dp.folder_name prefix = dp.solution.nickname + "-" if raw_folder.startswith(prefix): - stripped_folder = raw_folder[len(prefix):] + stripped_folder = raw_folder[len(prefix) :] else: stripped_folder = raw_folder folder_relative_path = subfolder / stripped_folder @@ -2364,7 +2364,11 @@ def format_filter(dps: list[Deployment], type_): for dp in sorted_filter: subfolder = SUBFOLDER_MAP.get(dp.solution.nickname, Path("")) prefix = dp.solution.nickname + "-" - folder = dp.folder_name[len(prefix):] if dp.folder_name.startswith(prefix) else dp.folder_name + folder = ( + dp.folder_name[len(prefix) :] + if dp.folder_name.startswith(prefix) + else dp.folder_name + ) names.append( f"[{dp.name}-{dp.solution.nickname}]({REPO_URL}/tree/main/{subfolder}/{folder})" ) diff --git a/image/sana/sana-1600m/packages/Sana/diffusion/model/dpm_solver.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/dpm_solver.py index 826c373ec..a791dcd38 100755 --- a/image/sana/sana-1600m/packages/Sana/diffusion/model/dpm_solver.py +++ b/image/sana/sana-1600m/packages/Sana/diffusion/model/dpm_solver.py @@ -1446,8 +1446,8 @@ def dpm_solver_adaptive( lower_update = lambda x, s, t: self.dpm_solver_first_update( x, s, t, return_intermediate=True ) - higher_update = ( - lambda x, s, t, **kwargs: self.singlestep_dpm_solver_second_update( + higher_update = lambda x, s, t, **kwargs: ( + self.singlestep_dpm_solver_second_update( x, s, t, r1=r1, solver_type=solver_type, **kwargs ) ) @@ -1456,8 +1456,8 @@ def dpm_solver_adaptive( lower_update = lambda x, s, t: self.singlestep_dpm_solver_second_update( x, s, t, r1=r1, return_intermediate=True, solver_type=solver_type ) - higher_update = ( - lambda x, s, t, **kwargs: self.singlestep_dpm_solver_third_update( + higher_update = lambda x, s, t, **kwargs: ( + self.singlestep_dpm_solver_third_update( x, s, t, r1=r1, r2=r2, solver_type=solver_type, **kwargs ) ) diff --git a/image/sana/sana-1600m/packages/Sana/diffusion/model/edm_sample.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/edm_sample.py index 9e99301eb..77d44a40d 100755 --- a/image/sana/sana-1600m/packages/Sana/diffusion/model/edm_sample.py +++ b/image/sana/sana-1600m/packages/Sana/diffusion/model/edm_sample.py @@ -128,22 +128,17 @@ def ablation_sampler( assert scaling in ["vp", "none"] # Helper functions for VP & VE noise level schedules. - vp_sigma = ( - lambda beta_d, beta_min: lambda t: ( - np.e ** (0.5 * beta_d * (t**2) + beta_min * t) - 1 - ) - ** 0.5 + vp_sigma = lambda beta_d, beta_min: ( + lambda t: (np.e ** (0.5 * beta_d * (t**2) + beta_min * t) - 1) ** 0.5 ) - vp_sigma_deriv = ( - lambda beta_d, beta_min: lambda t: 0.5 - * (beta_min + beta_d * t) - * (sigma(t) + 1 / sigma(t)) + vp_sigma_deriv = lambda beta_d, beta_min: ( + lambda t: 0.5 * (beta_min + beta_d * t) * (sigma(t) + 1 / sigma(t)) ) - vp_sigma_inv = ( - lambda beta_d, beta_min: lambda sigma: ( - (beta_min**2 + 2 * beta_d * (sigma**2 + 1).log()).sqrt() - beta_min + vp_sigma_inv = lambda beta_d, beta_min: ( + lambda sigma: ( + ((beta_min**2 + 2 * beta_d * (sigma**2 + 1).log()).sqrt() - beta_min) + / beta_d ) - / beta_d ) ve_sigma = lambda t: t.sqrt() ve_sigma_deriv = lambda t: 0.5 / t.sqrt() diff --git a/image/sana/sana-1600m/packages/Sana/diffusion/model/sa_solver.py b/image/sana/sana-1600m/packages/Sana/diffusion/model/sa_solver.py index b7566cef5..f36ccb4d0 100755 --- a/image/sana/sana-1600m/packages/Sana/diffusion/model/sa_solver.py +++ b/image/sana/sana-1600m/packages/Sana/diffusion/model/sa_solver.py @@ -202,10 +202,8 @@ def inverse_lambda(self, lamb): return t.reshape((-1,)) else: log_alpha = -0.5 * torch.logaddexp(-2.0 * lamb, torch.zeros((1,)).to(lamb)) - t_fn = ( - lambda log_alpha_t: torch.arccos( - torch.exp(log_alpha_t + self.cosine_log_alpha_0) - ) + t_fn = lambda log_alpha_t: ( + torch.arccos(torch.exp(log_alpha_t + self.cosine_log_alpha_0)) * 2.0 * (1.0 + self.cosine_s) / math.pi diff --git a/image/sana/sana-600m/packages/Sana/diffusion/model/dpm_solver.py b/image/sana/sana-600m/packages/Sana/diffusion/model/dpm_solver.py index 826c373ec..a791dcd38 100755 --- a/image/sana/sana-600m/packages/Sana/diffusion/model/dpm_solver.py +++ b/image/sana/sana-600m/packages/Sana/diffusion/model/dpm_solver.py @@ -1446,8 +1446,8 @@ def dpm_solver_adaptive( lower_update = lambda x, s, t: self.dpm_solver_first_update( x, s, t, return_intermediate=True ) - higher_update = ( - lambda x, s, t, **kwargs: self.singlestep_dpm_solver_second_update( + higher_update = lambda x, s, t, **kwargs: ( + self.singlestep_dpm_solver_second_update( x, s, t, r1=r1, solver_type=solver_type, **kwargs ) ) @@ -1456,8 +1456,8 @@ def dpm_solver_adaptive( lower_update = lambda x, s, t: self.singlestep_dpm_solver_second_update( x, s, t, r1=r1, return_intermediate=True, solver_type=solver_type ) - higher_update = ( - lambda x, s, t, **kwargs: self.singlestep_dpm_solver_third_update( + higher_update = lambda x, s, t, **kwargs: ( + self.singlestep_dpm_solver_third_update( x, s, t, r1=r1, r2=r2, solver_type=solver_type, **kwargs ) ) diff --git a/image/sana/sana-600m/packages/Sana/diffusion/model/edm_sample.py b/image/sana/sana-600m/packages/Sana/diffusion/model/edm_sample.py index 9e99301eb..77d44a40d 100755 --- a/image/sana/sana-600m/packages/Sana/diffusion/model/edm_sample.py +++ b/image/sana/sana-600m/packages/Sana/diffusion/model/edm_sample.py @@ -128,22 +128,17 @@ def ablation_sampler( assert scaling in ["vp", "none"] # Helper functions for VP & VE noise level schedules. - vp_sigma = ( - lambda beta_d, beta_min: lambda t: ( - np.e ** (0.5 * beta_d * (t**2) + beta_min * t) - 1 - ) - ** 0.5 + vp_sigma = lambda beta_d, beta_min: ( + lambda t: (np.e ** (0.5 * beta_d * (t**2) + beta_min * t) - 1) ** 0.5 ) - vp_sigma_deriv = ( - lambda beta_d, beta_min: lambda t: 0.5 - * (beta_min + beta_d * t) - * (sigma(t) + 1 / sigma(t)) + vp_sigma_deriv = lambda beta_d, beta_min: ( + lambda t: 0.5 * (beta_min + beta_d * t) * (sigma(t) + 1 / sigma(t)) ) - vp_sigma_inv = ( - lambda beta_d, beta_min: lambda sigma: ( - (beta_min**2 + 2 * beta_d * (sigma**2 + 1).log()).sqrt() - beta_min + vp_sigma_inv = lambda beta_d, beta_min: ( + lambda sigma: ( + ((beta_min**2 + 2 * beta_d * (sigma**2 + 1).log()).sqrt() - beta_min) + / beta_d ) - / beta_d ) ve_sigma = lambda t: t.sqrt() ve_sigma_deriv = lambda t: 0.5 / t.sqrt() diff --git a/image/sana/sana-600m/packages/Sana/diffusion/model/sa_solver.py b/image/sana/sana-600m/packages/Sana/diffusion/model/sa_solver.py index b7566cef5..f36ccb4d0 100755 --- a/image/sana/sana-600m/packages/Sana/diffusion/model/sa_solver.py +++ b/image/sana/sana-600m/packages/Sana/diffusion/model/sa_solver.py @@ -202,10 +202,8 @@ def inverse_lambda(self, lamb): return t.reshape((-1,)) else: log_alpha = -0.5 * torch.logaddexp(-2.0 * lamb, torch.zeros((1,)).to(lamb)) - t_fn = ( - lambda log_alpha_t: torch.arccos( - torch.exp(log_alpha_t + self.cosine_log_alpha_0) - ) + t_fn = lambda log_alpha_t: ( + torch.arccos(torch.exp(log_alpha_t + self.cosine_log_alpha_0)) * 2.0 * (1.0 + self.cosine_s) / math.pi diff --git a/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py index 922655006..06ac62a88 100644 --- a/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py +++ b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/models/diffusion/ddim.py @@ -38,8 +38,8 @@ def make_schedule( assert alphas_cumprod.shape[0] == self.ddpm_num_timesteps, ( "alphas have to be defined for each timestep" ) - to_torch = ( - lambda x: x.clone().detach().to(torch.float32).to(self.model.betas.device) + to_torch = lambda x: ( + x.clone().detach().to(torch.float32).to(self.model.betas.device) ) self.register_buffer("betas", to_torch(self.model.betas)) diff --git a/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/attention.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/attention.py index a5d68e8ea..d957f8acc 100644 --- a/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/attention.py +++ b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/attention.py @@ -220,11 +220,13 @@ def forward(self, x, context=None, mask=None): b, _, _ = q.shape q, k, v = map( - lambda t: t.unsqueeze(3) - .reshape(b, t.shape[1], self.heads, self.dim_head) - .permute(0, 2, 1, 3) - .reshape(b * self.heads, t.shape[1], self.dim_head) - .contiguous(), + lambda t: ( + t.unsqueeze(3) + .reshape(b, t.shape[1], self.heads, self.dim_head) + .permute(0, 2, 1, 3) + .reshape(b * self.heads, t.shape[1], self.dim_head) + .contiguous() + ), (q, k, v), ) diff --git a/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py index 2e0377430..79c4885ea 100644 --- a/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py +++ b/infrastructure/autodesk-wala/packages/src/mvdream/ldm/modules/diffusionmodules/model.py @@ -233,11 +233,13 @@ def forward(self, x): q, k, v = map(lambda x: rearrange(x, "b c h w -> b (h w) c"), (q, k, v)) q, k, v = map( - lambda t: t.unsqueeze(3) - .reshape(B, t.shape[1], 1, C) - .permute(0, 2, 1, 3) - .reshape(B * 1, t.shape[1], C) - .contiguous(), + lambda t: ( + t.unsqueeze(3) + .reshape(B, t.shape[1], 1, C) + .permute(0, 2, 1, 3) + .reshape(B * 1, t.shape[1], C) + .contiguous() + ), (q, k, v), ) out = xformers.ops.memory_efficient_attention( From 1c0044e745eaeed77cfb74a083857200039b941b Mon Sep 17 00:00:00 2001 From: Pat Date: Thu, 12 Feb 2026 16:14:35 -0600 Subject: [PATCH 3/4] remove _archive --- _archive/README.md | 11 - _archive/deepspeed-mii/README.md | 76 ------ _archive/deepspeed-mii/config.yaml | 30 --- _archive/deepspeed-mii/model/__init__.py | 0 _archive/deepspeed-mii/model/model.py | 79 ------- _archive/model_cach_gcs/config.yaml | 21 -- _archive/model_cach_gcs/model/model.py | 32 --- .../nous-capybara-34b-openai/README.md | 75 ------ .../nous-capybara-34b-openai/config.yaml | 24 -- .../model/__init__.py | 0 .../nous-capybara-34b-openai/model/model.py | 115 --------- .../nous-capybara/nous-capybara-34b/README.md | 97 -------- .../nous-capybara-34b/config.yaml | 22 -- .../nous-capybara-34b/model/__init__.py | 0 .../nous-capybara-34b/model/model.py | 98 -------- _archive/vllm/README.md | 213 ----------------- _archive/vllm/config.yaml | 19 -- _archive/vllm/model/__init__.py | 0 _archive/vllm/model/helper.py | 57 ----- _archive/vllm/model/model.py | 220 ------------------ _archive/whisper-torchserve/README.md | 82 ------- _archive/whisper-torchserve/config.yaml | 24 -- .../whisper-torchserve/data/config.properties | 21 -- _archive/whisper-torchserve/model/__init__.py | 0 _archive/whisper-torchserve/model/model.py | 69 ------ 25 files changed, 1385 deletions(-) delete mode 100644 _archive/README.md delete mode 100644 _archive/deepspeed-mii/README.md delete mode 100644 _archive/deepspeed-mii/config.yaml delete mode 100644 _archive/deepspeed-mii/model/__init__.py delete mode 100644 _archive/deepspeed-mii/model/model.py delete mode 100644 _archive/model_cach_gcs/config.yaml delete mode 100644 _archive/model_cach_gcs/model/model.py delete mode 100644 _archive/nous-capybara/nous-capybara-34b-openai/README.md delete mode 100644 _archive/nous-capybara/nous-capybara-34b-openai/config.yaml delete mode 100644 _archive/nous-capybara/nous-capybara-34b-openai/model/__init__.py delete mode 100644 _archive/nous-capybara/nous-capybara-34b-openai/model/model.py delete mode 100644 _archive/nous-capybara/nous-capybara-34b/README.md delete mode 100644 _archive/nous-capybara/nous-capybara-34b/config.yaml delete mode 100644 _archive/nous-capybara/nous-capybara-34b/model/__init__.py delete mode 100644 _archive/nous-capybara/nous-capybara-34b/model/model.py delete mode 100644 _archive/vllm/README.md delete mode 100644 _archive/vllm/config.yaml delete mode 100644 _archive/vllm/model/__init__.py delete mode 100644 _archive/vllm/model/helper.py delete mode 100644 _archive/vllm/model/model.py delete mode 100644 _archive/whisper-torchserve/README.md delete mode 100644 _archive/whisper-torchserve/config.yaml delete mode 100644 _archive/whisper-torchserve/data/config.properties delete mode 100644 _archive/whisper-torchserve/model/__init__.py delete mode 100644 _archive/whisper-torchserve/model/model.py diff --git a/_archive/README.md b/_archive/README.md deleted file mode 100644 index c6a0c8ab5..000000000 --- a/_archive/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Archive - -These examples are deprecated or outdated. See the current examples in the parent directory. - -| Directory | Description | -|-----------|-------------| -| [deepspeed-mii](deepspeed-mii/) | DeepSpeed MII inference (deprecated) | -| [model_cach_gcs](model_cach_gcs/) | GCS model caching (deprecated) | -| [nous-capybara](nous-capybara/) | Nous Capybara model (deprecated) | -| [vllm](vllm/) | Legacy vLLM example (deprecated) | -| [whisper-torchserve](whisper-torchserve/) | Whisper speech-to-text via TorchServe backend (deprecated) | diff --git a/_archive/deepspeed-mii/README.md b/_archive/deepspeed-mii/README.md deleted file mode 100644 index 2322c411b..000000000 --- a/_archive/deepspeed-mii/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# Llama-2-chat 7B DeepSpeed MII Truss - -This is a [Truss](https://truss.baseten.co/) for Llama-2-chat 7B served with [DeepSpeed MII](https://github.com/microsoft/DeepSpeed-MII). Llama 2 is a family of language models released by Meta. This README will walk you through how to deploy this Truss on Baseten to get your own instance of Llama-2-chat 7B. - -## Truss - -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp)). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - -### Get Llama 2 access - -Llama 2 currently requires approval to access. To request access: - -1. Go to [https://ai.meta.com/resources/models-and-libraries/llama-downloads/](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) and request access using the email associated with your HuggingFace account. -2. Go to [https://huggingface.co/meta-llama/Llama-2-7b](https://huggingface.co/meta-llama/Llama-2-7b) and request access. - -Once you have Llama access: - -1. Create a [HuggingFace access token](https://huggingface.co/settings/tokens) -2. Set it as a [secret in your Baseten account](https://app.baseten.co/settings/secrets) with the name `hf_access_token` - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd deepspeed-mii -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `deepspeed-mii` as your working directory, you can deploy the model with: - -```sh -truss push --trusted -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -### Hardware notes - -This seven billion parameter model is running in `float16` so that it fits on an A10G. - -## Llama-2-chat 7B API documentation - -This section provides an overview of the Llama-2-chat 7B API, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __max_tokens__ (optional, default=512): The maximum number of tokens to return, counting input tokens. Maximum of 4096. - -## Example usage - -```sh -truss predict -d '{"prompt": "What is the meaning of life?", "max_tokens": 1024}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "What's the meaning of life?", - "max_tokens": 1024 - }' -``` diff --git a/_archive/deepspeed-mii/config.yaml b/_archive/deepspeed-mii/config.yaml deleted file mode 100644 index 44f423609..000000000 --- a/_archive/deepspeed-mii/config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -description: Generate text from a prompt with this seven billion parameter language - model. -environment_variables: {} -external_package_dirs: [] -model_metadata: - avatar_url: https://cdn.baseten.co/production/static/explore/meta.png - cover_image_url: https://cdn.baseten.co/production/static/explore/llama.png - example_model_input: - max_tokens: 1024 - prompt: What's the meaning of life? - max_length: 4096 - repo_id: meta-llama/Llama-2-7b-chat-hf - tags: - - text-generation - tensor_parallel: 1 -model_name: Llama-2-chat 7B DeepSpeed MII -python_version: py311 -requirements: -- deepspeed-mii==0.1.1 -resources: - accelerator: A100 - cpu: '3' - memory: 14Gi - use_gpu: true -runtime: - predict_concurrency: 256 -secrets: - hf_access_token: null -system_packages: -- cuda-toolkit-12-2 diff --git a/_archive/deepspeed-mii/model/__init__.py b/_archive/deepspeed-mii/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/_archive/deepspeed-mii/model/model.py b/_archive/deepspeed-mii/model/model.py deleted file mode 100644 index 50d4e45eb..000000000 --- a/_archive/deepspeed-mii/model/model.py +++ /dev/null @@ -1,79 +0,0 @@ -import asyncio -import logging -import queue -import threading -from typing import Any, Dict - -import mii -from huggingface_hub import login - -DEFAULT_RESPONSE_MAX_LENGTH = 512 - - -class Model: - def __init__(self, **kwargs) -> None: - self.hf_access_token = kwargs["secrets"]["hf_access_token"] - model_metadata = kwargs["config"]["model_metadata"] - self.repo_id = model_metadata["repo_id"] - self.max_length = int(model_metadata["max_length"]) - self.tensor_parallel = int(model_metadata["tensor_parallel"]) - self.is_live_reload = kwargs["config"]["live_reload"] - - def load(self): - login(token=self.hf_access_token) - # need to create a new loop because `mii.serve` creates async client at the end, - # and `load` function being called from new thread - asyncio.set_event_loop(asyncio.new_event_loop()) - try: - mii.serve( - self.repo_id, - tensor_parallel=self.tensor_parallel, - max_length=self.max_length, - ) - except Exception as e: - if self.is_live_reload: - # in live reload `mii.serve` fails after reload since server is running already - logging.info( - "An exception occurred while starting mii server: %s, ignoring the exception due to live reload enabled", - e, - ) - else: - raise e - - def predict(self, request: Dict): - prompt = request.pop("prompt") - generate_args = { - "max_new_tokens": request.pop("max_tokens", DEFAULT_RESPONSE_MAX_LENGTH), - "ignore_eos": request.pop("ignore_eos", False), - } - - if request.pop("stream", False): - return self.stream(prompt, generate_args) - else: - # we need to create new asyncio loop because each request is being server from new thread - new_loop = asyncio.new_event_loop() - asyncio.set_event_loop(new_loop) - client = mii.client(self.repo_id) - response = client.generate(prompt, **generate_args) - new_loop.close() - - return {"text": "\n".join(response.response)} - - def stream(self, prompt: str, generate_args: Dict[str, Any]): - q = queue.Queue() - - def generate(): - new_loop = asyncio.new_event_loop() - asyncio.set_event_loop(new_loop) - client = mii.client(self.repo_id) - client.generate(prompt, streaming_fn=q.put, **generate_args) - q.put(None) - - threading.Thread(target=generate).start() - - while True: - item = q.get() # This will block until an item is available - if item is not None: - yield "".join(item.response) - else: - break diff --git a/_archive/model_cach_gcs/config.yaml b/_archive/model_cach_gcs/config.yaml deleted file mode 100644 index 953ca8e2c..000000000 --- a/_archive/model_cach_gcs/config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -model_name: Hello Model Cache GCS -python_version: py311 -requirements: [""] -resources: - accelerator: null - cpu: "1" - memory: 2Gi - use_gpu: false -build: - secret_to_path_mapping: - gcs-service-account-jsn: /secrets/gcs-service-account-jsn -secrets: { gcs-service-account-jsn: null } # null is encouraged, as this will automatically use the one provided by baseten.co -model_cache: - # this is a simple private bucket, uploaded llama-3-2, as is from huggingface. - # The repo contains e.g. gs://llama-3-2-1b-instruct/config.json / gs://llama-3-2-1b-instruct/model.safetensors - - repo_id: gs://llama-3-2-1b-instruct/ - revision: main - use_volume: true - volume_folder: llama - runtime_secret_name: "gcs-service-account-jsn" - kind: "gcs" diff --git a/_archive/model_cach_gcs/model/model.py b/_archive/model_cach_gcs/model/model.py deleted file mode 100644 index e949d452b..000000000 --- a/_archive/model_cach_gcs/model/model.py +++ /dev/null @@ -1,32 +0,0 @@ -# <- download is invoked before here. -import pathlib - - -class Model: - """example usage of `model_cache` in truss""" - - def __init__(self, *args, **kwargs): - # `lazy_data_resolver` is passed as keyword-argument in init - self._lazy_data_resolver = kwargs["lazy_data_resolver"] - self.tensor_size = None - - def load(self): - # work that does not require the download may be done beforehand - # important to collect the download before using any incomplete data - self._lazy_data_resolver.block_until_download_complete() - # after the call, you may use the /app/model_cache directory and the contents - # torch.load( - # "/app/model_cache/stable-diffusion-xl-base/vae_1_0/diffusion_pytorch_model.fp16.safetensors", - # weights_only=True - # ) - self.tensor_size = ( - pathlib.Path("/app/model_cache/llama/model.safetensors").stat().st_size - ) - print( - "Model loaded successfully with size of {} bytes".format(self.tensor_size) - ) - - def predict(self, input_data): - # this method will be called by the serving container - # you may use the model here, after the download is complete - return {"input": input_data, "tensor_size": self.tensor_size} diff --git a/_archive/nous-capybara/nous-capybara-34b-openai/README.md b/_archive/nous-capybara/nous-capybara-34b-openai/README.md deleted file mode 100644 index a37905174..000000000 --- a/_archive/nous-capybara/nous-capybara-34b-openai/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Nous Capybara 34B Truss (OpenAI Client Compatible) - -This is a [Truss](https://truss.baseten.co/) for [Nous Capybara 34B](https://huggingface.co/NousResearch/Nous-Capybara-34B), compatible with our [bridge endpoint for OpenAI ChatCompletion users](https://docs.baseten.co/api-reference/openai). - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd nous-capybara/nous-capybara-34b-openai -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `nous-capybara/nous-capybara-34b-openai` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Model API reference - -This model is designed for our ChatCompletions endpoint: - -- [ChatCompletions endpoint tutorial](https://www.baseten.co/blog/gpt-vs-mistral-migrate-to-open-source-llms-with-minor-code-changes/) -- [ChatCompletions endpoint reference docs](https://docs.baseten.co/api-reference/openai) - -Note that Nous Capybara currently does not support system messages (see [here](https://huggingface.co/NousResearch/Nous-Capybara-34B/discussions/5)). - -See the script below for an example of calling the model, with and without streaming: -```python -from openai import OpenAI -import os - -model_id = "YOUR_MODEL_ID" -client = OpenAI( - api_key="YOUR_BASETEN_API_KEY", - base_url=f"https://bridge.baseten.co/{model_id}/v1" -) - -# Non-streaming example -response = client.chat.completions.create( - model="nous-capybara-34b", - messages=[ - {"role": "user", "content": "What happens if I go to the top of the tallest mountain in California with a bucket of water and tip it over the highest cliff?"}, - ], - stream=False, -) - -print(response.choices[0].message.content) - -# Streaming example -response = client.chat.completions.create( - model="nous-capybara-34b", - messages=[ - {"role": "user", "content": "Who won the world series in 2020?"}, - {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."}, - {"role": "user", "content": "Where was it played?"} - ], - stream=True, -) - -for chunk in response: - content = chunk.choices[0].delta.content - if content: - print(content, end="") -``` diff --git a/_archive/nous-capybara/nous-capybara-34b-openai/config.yaml b/_archive/nous-capybara/nous-capybara-34b-openai/config.yaml deleted file mode 100644 index f071d9d60..000000000 --- a/_archive/nous-capybara/nous-capybara-34b-openai/config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_cache: -- allow_patterns: - - '*.json' - - '*.bin' - repo_id: NousResearch/Nous-Capybara-34B - use_volume: false -model_name: Nous Capybara 34B OpenAI -python_version: py310 -requirements: -- accelerate==0.25.0 -- transformers==4.35.2 -- torch==2.1.0 -- bitsandbytes==0.41.3 -- scipy==1.11.4 -- sentencepiece==0.1.99 -resources: - accelerator: A100 - cpu: '3' - memory: 20Gi - use_gpu: true -secrets: {} -system_packages: [] diff --git a/_archive/nous-capybara/nous-capybara-34b-openai/model/__init__.py b/_archive/nous-capybara/nous-capybara-34b-openai/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/_archive/nous-capybara/nous-capybara-34b-openai/model/model.py b/_archive/nous-capybara/nous-capybara-34b-openai/model/model.py deleted file mode 100644 index d0d124d00..000000000 --- a/_archive/nous-capybara/nous-capybara-34b-openai/model/model.py +++ /dev/null @@ -1,115 +0,0 @@ -from threading import Thread - -import torch -from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - GenerationConfig, - TextIteratorStreamer, -) - -MODEL_NAME = "NousResearch/Nous-Capybara-34B" - -DEFAULT_STREAM = False - - -def _format_prompt(messages: list[dict], add_generation_prompt: bool = True) -> str: - """Given a list of messages in the form: [{'role': 'user', 'content': 'hello world'}], returns - the messages as a string in the form that Nous-Capybara expects: - 'USER: hello world - ASSISTANT:' - - This is a workaround for Nous-Capybara not being configured with a chat template (see https://huggingface.co/NousResearch/Nous-Capybara-34B/discussions/5) - """ - formatted_prompts = [] - for message in messages: - if message["role"] == "user": - formatted_prompts.append(f"USER: {message['content']}") - elif message["role"] == "assistant": - formatted_prompts.append(f"ASSISTANT: {message['content']}") - # Note: Capybara doesn't support system messages. See https://huggingface.co/NousResearch/Nous-Capybara-34B/discussions/8 - if add_generation_prompt: - formatted_prompts.append("ASSISTANT:") - - return "\n".join(formatted_prompts) - - -class Model: - def __init__(self, **kwargs): - self.model = None - self.tokenizer = None - - def load(self): - self.model = AutoModelForCausalLM.from_pretrained( - MODEL_NAME, - device_map="auto", - torch_dtype=torch.float16, - trust_remote_code=True, - ) - self.tokenizer = AutoTokenizer.from_pretrained( - MODEL_NAME, trust_remote_code=True - ) - - def preprocess(self, request: dict): - # Set generate_args to default values. - default_generate_args = { - "max_new_tokens": 256, - "temperature": 0.7, - "top_p": 0.8, - "top_k": 40, - "repetition_penalty": 1.3, - "no_repeat_ngram_size": 5, - "use_cache": True, - "do_sample": True, - "eos_token_id": self.tokenizer.eos_token_id, - "pad_token_id": self.tokenizer.pad_token_id, - } - request["generate_args"] = default_generate_args - - # Override generate_args with values provided in the user's request. - for k in default_generate_args: - if k in request and request[k] is not None: - request["generate_args"][k] = request[k] - - return request - - def stream(self, input_ids: list, generation_args: dict): - streamer = TextIteratorStreamer(self.tokenizer) - generation_config = GenerationConfig(**generation_args) - generation_kwargs = { - "input_ids": input_ids, - "generation_config": generation_config, - "return_dict_in_generate": True, - "output_scores": True, - "max_new_tokens": generation_args["max_new_tokens"], - "streamer": streamer, - } - - with torch.no_grad(): - # Begin generation in a separate thread - thread = Thread(target=self.model.generate, kwargs=generation_kwargs) - thread.start() - - # Yield generated text as it becomes available - def inner(): - for text in streamer: - yield text - thread.join() - - return inner() - - def predict(self, request: dict): - stream = request.pop("stream", DEFAULT_STREAM) - messages = request.pop("messages") - input_ids = self.tokenizer( - _format_prompt(messages), return_tensors="pt" - ).input_ids.cuda() - - generation_args = request.pop("generate_args") - - if stream: - return self.stream(input_ids, generation_args) - - with torch.no_grad(): - outputs = self.model.generate(inputs=input_ids, **generation_args) - return self.tokenizer.decode(outputs[0], skip_special_tokens=True) diff --git a/_archive/nous-capybara/nous-capybara-34b/README.md b/_archive/nous-capybara/nous-capybara-34b/README.md deleted file mode 100644 index cf72d77f6..000000000 --- a/_archive/nous-capybara/nous-capybara-34b/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Nous Capybara 34B Truss - -This is a [Truss](https://truss.baseten.co/) for [Nous Capybara 34B](https://huggingface.co/NousResearch/Nous-Capybara-34B). This model is a fine-tuned version of Yi-34B with a 200K context length. - -## Truss -Truss is an open-source model serving framework developed by Baseten. It allows you to develop and deploy machine learning models onto Baseten (and other platforms like [AWS](https://truss.baseten.co/deploy/aws) or [GCP](https://truss.baseten.co/deploy/gcp). Using Truss, you can develop a GPU model using [live-reload](https://baseten.co/blog/technical-deep-dive-truss-live-reload), package models and their associated code, create Docker containers and deploy on Baseten. - - -## Deployment - -First, clone this repository: - -```sh -git clone https://github.com/basetenlabs/truss-examples/ -cd nous-capybara/nous-capybara-34b -``` - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `nous-capybara/nous-capybara-34b` as your working directory, you can deploy the model with: - -```sh -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - - -### Hardware notes - -This 34 billion parameter model requires an A100 GPU. - -## Nous Capybara 34B Chat API documentation - -This section provides an overview of the Nous-Capybara-34B model, its parameters, and how to use it. The API consists of a single route named `predict`, which you can invoke to generate text based on the provided prompt. - -### API route: `predict` - -The predict route is the primary method for generating text completions based on a given prompt. It takes several parameters: - -- __prompt__: The input text that you want the model to generate a response for. -- __max_tokens__ (optional, default=256): The maximum number of tokens to return, counting input tokens. -- __temperature__ (optional, default=0.7): Controls the randomness of the generated text. Higher values produce more diverse results, while lower values produce more deterministic results. -- __top_p__ (optional, default=0.8): The cumulative probability threshold for token sampling. The model will only consider tokens whose cumulative probability is below this threshold. -- __top_k__ (optional, default=40): The number of top tokens to consider when sampling. The model will only consider the top_k highest-probability tokens. -- __repetition_penalty__ (optional, default=1.3): Helps the model generate more diverse content instead of repeating previous phrases. -- __no_repeat_ngram_size__ (optional, default=5): Specifies the length of token sets that are completely blocked from repeating at all. -- __stream__ (optional, default=True): Allows you to receive the tokens are they are generated in a stream like fashion. - -The API also supports passing any parameter supported by HuggingFace's `Transformers.generate`. - -The output of the model is a JSON object which only have one key called `output`. Here is an example of what that JSON object looks like: -```json -{"output": "There's a place where time stands still. A place of breath taking wonder, but also deep mystery and danger; the ocean floor.."} -``` - -## Example usage - -```sh -truss predict -d '{"prompt": "There is a place where time stands still. A place of breath taking wonder, but also", "max_tokens": 512}' -``` - -You can also invoke your model via a REST API: - -``` -curl -X POST " https://app.baseten.co/model_versions/YOUR_MODEL_VERSION_ID/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {YOUR_API_KEY}' \ - -d '{ - "prompt": "What happens if I go to the top of the tallest mountian in california with a bucket of water and tip it over the highest cliff?", - "max_tokens": 512, - "stream": False - }' -``` - -By default streaming the tokens is enabled. Here is an example of how to invoke the model with streaming in Python: -```python -import requests -headers = {"Authorization": f"Api-Key BASETEN-API-KEY"} - -res = requests.post( - "https://model-.api.baseten.co/development/predict", - headers=headers, - json={"prompt": "What happens if I go to the top of the tallest mountian in california with a bucket of water and tip it over the highest cliff?", - "max_tokens": 512, "temperature": 0.9, "stream": True}, - stream=True -) -res.raise_for_status() - -for word in res: - print(word.decode("utf-8")) -``` diff --git a/_archive/nous-capybara/nous-capybara-34b/config.yaml b/_archive/nous-capybara/nous-capybara-34b/config.yaml deleted file mode 100644 index fc567d33e..000000000 --- a/_archive/nous-capybara/nous-capybara-34b/config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_metadata: - example_model_input: - prompt: What happens if I go to the top of the tallest mountian in california - with a bucket of water and tip it over the highest cliff? -model_name: Nous Capybara 34B -python_version: py310 -requirements: -- accelerate==0.25.0 -- transformers==4.35.2 -- torch==2.1.0 -- bitsandbytes==0.41.3 -- scipy==1.11.4 -- sentencepiece==0.1.99 -resources: - accelerator: A100 - cpu: '3' - memory: 20Gi - use_gpu: true -secrets: {} -system_packages: [] diff --git a/_archive/nous-capybara/nous-capybara-34b/model/__init__.py b/_archive/nous-capybara/nous-capybara-34b/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/_archive/nous-capybara/nous-capybara-34b/model/model.py b/_archive/nous-capybara/nous-capybara-34b/model/model.py deleted file mode 100644 index 0b8f91bff..000000000 --- a/_archive/nous-capybara/nous-capybara-34b/model/model.py +++ /dev/null @@ -1,98 +0,0 @@ -from threading import Thread -from typing import Dict - -import torch -from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - GenerationConfig, - TextIteratorStreamer, -) - -MODEL_NAME = "NousResearch/Nous-Capybara-34B" -MAX_LENGTH = 256 -DO_SAMPLE = True -REPETITION_PENALTY = 1.3 -NO_REPEAT_NGRAM_SIZE = 5 -TEMPERATURE = 0.7 -TOP_K = 40 -TOP_P = 0.8 -DEFAULT_STREAM = True - - -class Model: - def __init__(self, **kwargs): - self.model = None - self.tokenizer = None - - def load(self): - self.model = AutoModelForCausalLM.from_pretrained( - MODEL_NAME, - device_map="auto", - torch_dtype=torch.float16, - trust_remote_code=True, - ) - self.tokenizer = AutoTokenizer.from_pretrained( - MODEL_NAME, trust_remote_code=True - ) - - def preprocess(self, request: dict): - generate_args = { - "max_length": request.get("max_tokens", MAX_LENGTH), - "temperature": request.get("temperature", TEMPERATURE), - "top_p": request.get("top_p", TOP_P), - "top_k": request.get("top_k", TOP_K), - "repetition_penalty": request.get("repetition_penalty", REPETITION_PENALTY), - "no_repeat_ngram_size": request.get( - "no_repeat_ngram_size", NO_REPEAT_NGRAM_SIZE - ), - "do_sample": request.get("do_sample", DO_SAMPLE), - "use_cache": True, - "eos_token_id": self.tokenizer.eos_token_id, - "pad_token_id": self.tokenizer.pad_token_id, - } - request["generate_args"] = generate_args - return request - - def stream(self, input_ids: list, generation_args: dict): - streamer = TextIteratorStreamer(self.tokenizer) - generation_config = GenerationConfig(**generation_args) - generation_kwargs = { - "input_ids": input_ids, - "generation_config": generation_config, - "return_dict_in_generate": True, - "output_scores": True, - "max_new_tokens": generation_args["max_length"], - "streamer": streamer, - } - - with torch.no_grad(): - # Begin generation in a separate thread - thread = Thread(target=self.model.generate, kwargs=generation_kwargs) - thread.start() - - # Yield generated text as it becomes available - def inner(): - for text in streamer: - yield text - thread.join() - - return inner() - - def predict(self, model_input: Dict): - prompt = model_input.get("prompt") - stream = model_input.get("stream", DEFAULT_STREAM) - generation_args = model_input.pop("generate_args") - - formatted_prompt = f"USER: {prompt}\n ASSISTANT:" - input_ids = self.tokenizer( - formatted_prompt, return_tensors="pt" - ).input_ids.cuda() - - if stream: - return self.stream(input_ids, generation_args) - - with torch.no_grad(): - outputs = self.model.generate(inputs=input_ids, **generation_args) - model_output = self.tokenizer.decode(outputs[0], skip_special_tokens=True) - return {"output": model_output} diff --git a/_archive/vllm/README.md b/_archive/vllm/README.md deleted file mode 100644 index 17fc3f903..000000000 --- a/_archive/vllm/README.md +++ /dev/null @@ -1,213 +0,0 @@ -# vLLM Truss to deploy chat completion model - -## What is this Truss example doing - -This is a general purpose [Truss](https://truss.baseten.co/) that can deploy an asynchronous vLLM engine([AsyncLLMEngine](https://docs.vllm.ai/en/latest/dev/engine/async_llm_engine.html#asyncllmengine)) of any customized configuration with [all compatible chat completion models](https://docs.vllm.ai/en/latest/models/supported_models.html). We create this example to give you the most codeless experience, so you can configure all vLLM engine parameters in `config.yaml`, without making code changes in `model.py` for most of the use cases. - -## Configure your Truss by modifying the config.yaml - -### Basic options using 1 GPU - -Here is the minimum config file you will need to deploy a model using vLLM on 1 GPU. -The only parameters you need to touch are: -- `model_name` -- `repo_id` -- `accelerator` - -``` -model_name: "Llama 3.1 8B Instruct VLLM" -python_version: py311 -model_metadata: - example_model_input: {"prompt": "what is the meaning of life"} - repo_id: meta-llama/Llama-3.1-8B-Instruct - openai_compatible: true - vllm_config: null -requirements: - - vllm==0.5.4 -resources: - accelerator: A100 - use_gpu: true -runtime: - predict_concurrency: 128 -secrets: - hf_access_token: null -``` - -### Basic options using multiple GPUs - -If your model needs more than 1 GPU to run using tensor parallel, you will need to change `accelerator`, and to set `tensor_parallel_size` and `distributed_executor_backend` accordingly. - -``` -model_name: "Llama 3.1 8B Instruct VLLM" -python_version: py311 -model_metadata: - example_model_input: {"prompt": "what is the meaning of life"} - repo_id: meta-llama/Llama-3.1-8B-Instruct - openai_compatible: false - vllm_config: - tensor_parallel_size: 4 - max_model_len: 4096 - distributed_executor_backend: mp -requirements: - - vllm==0.5.4 -resources: - accelerator: A10G:4 - use_gpu: true -runtime: - predict_concurrency: 128 -secrets: - hf_access_token: null -``` - -### Use vLLM's OpenAI compatible server - -To use vLLM in [OpenAI compatible server](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html) mode, simply set `openai_compatible: true` under `model_metadata`. - -### Customize vLLM engine parameters - -For advanced users who want to override [vLLM engine arguments](https://docs.vllm.ai/en/latest/models/engine_args.html), you can add all arguments to `vllm_config` under `model_metadata`. - -#### Example 1: using model quantization - -``` -model_name: Mistral 7B v2 vLLM AWQ - T4 -environment_variables: {} -external_package_dirs: [] -model_metadata: - repo_id: TheBloke/Mistral-7B-Instruct-v0.2-AWQ - vllm_config: - quantization: "awq" - dtype: "float16" - max_model_len: 8000 - max_num_seqs: 8 -python_version: py310 -requirements: - - vllm==0.5.4 -resources: - accelerator: T4 - use_gpu: true -secrets: - hf_access_token: null -system_packages: [] -runtime: - predict_concurrency: 128 -``` - -#### Example 2: using customized vLLM image - -You can even override with your own customized vLLM docker image to work with models that are not supported yet by vanilla vLLM. - -``` -model_name: Ultravox v0.2 -base_image: - image: vshulman/vllm-openai-fixie:latest - python_executable_path: /usr/bin/python3 -model_metadata: - repo_id: fixie-ai/ultravox-v0.2 - vllm_config: - audio_token_id: 128002 -environment_variables: {} -external_package_dirs: [] -python_version: py310 -runtime: - predict_concurrency: 512 -requirements: - - httpx -resources: - accelerator: A100 - use_gpu: true -secrets: - hf_access_token: null -system_packages: -- python3.10-venv -``` - -## Deploy your Truss - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` -3. With `vllm` as your working directory, you can deploy the model with: - - ```sh - truss push --trusted - ``` - - Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Call your model - -Once your deployment is up, there are [many ways](https://docs.baseten.co/invoke/quickstart) to call your model. - -### curl command - -#### If you are NOT using OpenAI compatible server - -``` -curl -X POST https://model-.api.baseten.co/development/predict \ - -H "Authorization: Api-Key $BASETEN_API_KEY" \ - -d '{"prompt": "what is the meaning of life"}' -``` - - -#### If you are using OpenAI compatible server - -``` -curl -X POST "https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {BASETEN_API_KEY}' \ - -d '{ - "messages": [{"role": "user", "content": "What even is AGI?"}], - "max_tokens": 256 - }' -``` - -To access [production metrics](https://docs.vllm.ai/en/latest/serving/metrics.html) reported by OpenAI compatible server, simply add `metrics: true` to the request. - -``` -curl -X POST "https://model-.api.baseten.co/development/predict" \ - -H "Content-Type: application/json" \ - -H 'Authorization: Api-Key {BASETEN_API_KEY}' \ - -d '{ - "metrics": true - }' -``` - -### OpenAI SDK (if you are using OpenAI compatible server) - -``` -from openai import OpenAI -import os - -model_id = "abcd1234" # Replace with your model ID -deployment_id = "4321cbda" # [Optional] Replace with your deployment ID - -client = OpenAI( - api_key=os.environ["BASETEN_API_KEY"], - base_url=f"https://bridge.baseten.co/{model_id}/v1/direct" -) - -response = client.chat.completions.create( - model="meta-llama/Llama-3.1-8B-Instruct", - messages=[ - {"role": "user", "content": "Who won the world series in 2020?"}, - {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."}, - {"role": "user", "content": "Where was it played?"} - ], - extra_body={ - "baseten": { - "model_id": model_id, - "deployment_id": deployment_id - } - } -) -print(response.choices[0].message.content) - -``` - -For more information, see [API reference](https://docs.baseten.co/api-reference/openai). - -## Support - -If you have any questions or need assistance, please open an issue in this repository or contact our support team. diff --git a/_archive/vllm/config.yaml b/_archive/vllm/config.yaml deleted file mode 100644 index bd98b7e40..000000000 --- a/_archive/vllm/config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -model_name: "Llama 3.1 8B Instruct VLLM openai compatible" -python_version: py311 -model_metadata: - example_model_input: {"prompt": "what is the meaning of life"} - repo_id: meta-llama/Llama-3.1-8B-Instruct - openai_compatible: true - vllm_config: - tensor_parallel_size: 1 - max_model_len: 4096 - enable_prefix_caching: true -requirements: - - vllm==0.5.4 -resources: - accelerator: A100 - use_gpu: true -runtime: - predict_concurrency: 128 -secrets: - hf_access_token: null diff --git a/_archive/vllm/model/__init__.py b/_archive/vllm/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/_archive/vllm/model/helper.py b/_archive/vllm/model/helper.py deleted file mode 100644 index 69067e11a..000000000 --- a/_archive/vllm/model/helper.py +++ /dev/null @@ -1,57 +0,0 @@ -import asyncio -import logging -import os -import threading - -import httpx - -logger = logging.getLogger(__name__) - -DEFAULT_HEALTH_CHECK_INTERVAL = 5 # seconds - - -async def monitor_vllm_server_health(vllm_server_url, health_check_interval): - assert vllm_server_url is not None, "vllm_server_url must not be None" - try: - async with httpx.AsyncClient() as client: - while True: - response = await client.get(f"{vllm_server_url}/health") - if response.status_code != 200: - raise RuntimeError("vLLM is unhealthy") - await asyncio.sleep(health_check_interval) - except Exception as e: - logging.error( - f"vLLM has gone into an unhealthy state due to error: {e}, restarting service now..." - ) - os._exit(1) - - -async def monitor_vllm_engine_health(vllm_engine, health_check_interval): - assert vllm_engine is not None, "vllm_engine must not be None" - try: - while True: - await vllm_engine.check_health() - await asyncio.sleep(health_check_interval) - except Exception as e: - logging.error( - f"vLLM has gone into an unhealthy state due to error: {e}, restarting service now..." - ) - os._exit(1) - - -def run_background_vllm_health_check( - use_openai_compatible_server=False, - health_check_interval=DEFAULT_HEALTH_CHECK_INTERVAL, - vllm_engine=None, - vllm_server_url=None, -): - logger.info("Starting background health check loop") - loop = asyncio.new_event_loop() - if use_openai_compatible_server: - loop.create_task( - monitor_vllm_server_health(vllm_server_url, health_check_interval) - ) - else: - loop.create_task(monitor_vllm_engine_health(vllm_engine, health_check_interval)) - thread = threading.Thread(target=loop.run_forever, daemon=True) - thread.start() diff --git a/_archive/vllm/model/model.py b/_archive/vllm/model/model.py deleted file mode 100644 index c6b5c1e16..000000000 --- a/_archive/vllm/model/model.py +++ /dev/null @@ -1,220 +0,0 @@ -import logging -import os -import subprocess -import time -import uuid - -import httpx -from model.helper import run_background_vllm_health_check -from transformers import AutoTokenizer -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.engine.async_llm_engine import AsyncLLMEngine - -from vllm import SamplingParams - -os.environ["TOKENIZERS_PARALLELISM"] = "true" - -logger = logging.getLogger(__name__) - - -class Model: - # TODO: better way to detect model server startup failure than using `MAX_FAILED_SECONDS` - MAX_FAILED_SECONDS = 1500 # 25 minutes; the reason this would take this long is mostly if we download a large model - HEALTH_CHECK_INTERVAL = 5 # seconds - - def __init__(self, **kwargs): - self._config = kwargs["config"] - self.model_id = None - self.llm_engine = None - self.model_args = None - self.hf_secret_token = kwargs["secrets"].get("hf_access_token", None) - self.openai_compatible = self._config["model_metadata"].get( - "openai_compatible", False - ) - self.vllm_base_url = None - os.environ["HF_TOKEN"] = self.hf_secret_token - - def load(self): - self._model_metadata = self._config["model_metadata"] - self._model_repo_id = self._model_metadata["repo_id"] - self._vllm_config = self._model_metadata["vllm_config"] - if self._vllm_config is None: - self._vllm_config = {} - logger.info(f"main model: {self._model_repo_id}") - logger.info(f"vllm config: {self._vllm_config}") - if self.openai_compatible: - self._client = httpx.AsyncClient(timeout=None) - command = ["vllm", "serve", self._model_repo_id] - for key, value in self._vllm_config.items(): - if value is True: - command.append(f"--{key.replace('_', '-')}") - elif value is False: - continue - else: - command.append(f"--{key.replace('_', '-')}") - command.append(str(value)) - - logger.info( - f"Starting openai compatible vLLM server with command: {command}" - ) - - self._vllm_process = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True - ) - - # Wait for 10 seconds and check if command fails - time.sleep(10) - - if self._vllm_process.poll() is None: - logger.info("Command to start vLLM server ran successfully") - else: - stdout, stderr = self._vllm_process.communicate() - if self._vllm_process.returncode != 0: - logger.error(f"Command failed with error: {stderr}") - raise RuntimeError( - f"Command failed with code {self._vllm_process.returncode}: {stderr}" - ) - - if self._vllm_config and "port" in self._vllm_config: - self._vllm_port = self._vllm_config["port"] - else: - self._vllm_port = 8000 - - self.vllm_base_url = f"http://localhost:{self._vllm_port}" - - # Polling to check if the server is up - server_up = False - start_time = time.time() - while time.time() - start_time < self.MAX_FAILED_SECONDS: - try: - response = httpx.get(f"{self.vllm_base_url}/health") - logger.info(f"Checking server health: {response.status_code}") - if response.status_code == 200: - server_up = True - break - except httpx.RequestError as e: - seconds_passed = int(time.time() - start_time) - if seconds_passed % 10 == 0: - logger.info( - f"Server is starting for {seconds_passed} seconds: {e}" - ) - time.sleep(1) # Wait for 1 second before retrying - - if not server_up: - raise RuntimeError( - "Server failed to start within the maximum allowed time." - ) - else: - try: - result = subprocess.run( - ["nvidia-smi"], capture_output=True, text=True, check=True - ) - logger.info(result.stdout) - except subprocess.CalledProcessError as e: - logger.error(f"Command failed with code {e.returncode}: {e.stderr}") - - self.model_args = AsyncEngineArgs( - model=self._model_repo_id, **self._vllm_config - ) - self.llm_engine = AsyncLLMEngine.from_engine_args( - engine_args=self.model_args - ) - self.tokenizer = AutoTokenizer.from_pretrained(self._model_repo_id) - - try: - result = subprocess.run( - ["nvidia-smi"], capture_output=True, text=True, check=True - ) - logger.info(result.stdout) - except subprocess.CalledProcessError as e: - logger.error(f"Command failed with code {e.returncode}: {e.stderr}") - try: - run_background_vllm_health_check( - self.openai_compatible, - self.HEALTH_CHECK_INTERVAL, - self.llm_engine, - self.vllm_base_url, - ) - except Exception as e: - raise RuntimeError(f"Failed to start background health check: {e}") - - async def predict(self, model_input): - if "messages" not in model_input and "prompt" not in model_input: - raise ValueError("Prompt or messages must be provided") - - stream = model_input.get("stream", False) - if self.openai_compatible: - # if the key metrics: true is present, let's return the vLLM /metrics endpoint - if model_input.get("metrics", False): - response = await self._client.get(f"{self.vllm_base_url}/metrics") - return response.text - - # convenience for Baseten bridge - if "model" not in model_input and self._model_repo_id: - logger.info( - f"model_input missing model due to Baseten bridge, using {self._model_repo_id}" - ) - model_input["model"] = self._model_repo_id - - if stream: - - async def generator(): - async with self._client.stream( - "POST", - f"{self.vllm_base_url}/v1/chat/completions", - json=model_input, - ) as response: - async for chunk in response.aiter_bytes(): - if chunk: - yield chunk - - return generator() - else: - response = await self._client.post( - f"{self.vllm_base_url}/v1/chat/completions", - json=model_input, - ) - return response.json() - else: - # SamplingParams does not take/use argument 'model' - if "model" in model_input: - model_input.pop("model") - if "prompt" in model_input: - prompt = model_input.pop("prompt") - sampling_params = SamplingParams(**model_input) - idx = str(uuid.uuid4().hex) - messages = [ - {"role": "user", "content": prompt}, - ] - # templatize the input to the model - input = self.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True - ) - elif "messages" in model_input: - messages = model_input.pop("messages") - sampling_params = SamplingParams(**model_input) - idx = str(uuid.uuid4().hex) - # templatize the input to the model - input = self.tokenizer.apply_chat_template( - messages, - tokenize=False, - ) - logger.info(f"Using SamplingParams: {sampling_params}") - # since we accept any valid vllm sampling parameters, we can just pass it through - vllm_generator = self.llm_engine.generate(input, sampling_params, idx) - - async def generator(): - full_text = "" - async for output in vllm_generator: - text = output.outputs[0].text - delta = text[len(full_text) :] - full_text = text - yield delta - - if stream: - return generator() - else: - full_text = "" - async for delta in generator(): - full_text += delta - return {"text": full_text} diff --git a/_archive/whisper-torchserve/README.md b/_archive/whisper-torchserve/README.md deleted file mode 100644 index e6f685b8b..000000000 --- a/_archive/whisper-torchserve/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Whisper Torchserve - -This truss allows you to run a whisper model using [torchserve](https://pytorch.org/serve/) as the backend on truss. - - -## Deployment - -Before deployment: - -1. Make sure you have a [Baseten account](https://app.baseten.co/signup) and [API key](https://app.baseten.co/settings/account/api_keys). -2. Install the latest version of Truss: `pip install --upgrade truss` - -With `whisper/whisper-torchserve` as your working directory, you can deploy the model with: - -``` -truss push -``` - -Paste your Baseten API key if prompted. - -For more information, see [Truss documentation](https://truss.baseten.co). - -## Model Inputs - -The model takes in one input: -- __audio__: An audio file as a base64 string - -## Few thing to note -Torchserve requires a compiled `.mar` file in order to serve the model. Here is a [README](https://github.com/pytorch/serve/blob/master/model-archiver/README.md) providing a brief explanation for generating this file. Once the `.mar` file is generated it needs to get placed in the `data/model_store` directory. Also in the `data/` directory is a configuration file for torchserve called `config.properties`. That file looks something like this: - -``` -inference_address=http://0.0.0.0:8888 -batch_size=4 -ipex_enable=true -async_logging=true - -models={\ - "whisper_base": {\ - "1.0": {\ - "defaultVersion": true,\ - "marName": "whisper_base.mar",\ - "minWorkers": 1,\ - "maxWorkers": 2,\ - "batchSize": 4,\ - "maxBatchDelay": 500,\ - "responseTimeout": 24\ - }\ - }\ -} -``` - -Here you can specify the `batchSize` as well as the name of your mar file using `marName`. When torchserve starts, it will looks for the mar file inside the `data/model_store` directory with the `marName` defined above. - -## Invoking the model - -Here is an example in Python: - -```python -import requests -import base64 - -def wav_to_base64(file_path): - with open(file_path, "rb") as wav_file: - binary_data = wav_file.read() - base64_data = base64.b64encode(binary_data) - base64_string = base64_data.decode("utf-8") - return base64_string - -resp = requests.post( - "https://model-.api.baseten.co/development/predict", - headers={"Authorization": "Api-Key BASETEN-API-KEY"}, - json={"audio": wav_to_base64("/path/to/audio-file/60-sec.wav")}, -) - -print(resp.json()) -``` - -Here is a sample output: - -```json -{"output": "Let me make it clear. His conduct is unacceptable. He's unfit. And be careful of what you're gonna get. He doesn't care for the American people. It's Donald Trump first. This is what I want people to understand. These people have... I mean, she has no idea what the hell the names of those provinces are, but she wants to send our sons and daughters and our troops and our military equipment to go fight it. Look at the blank expression. She doesn't know the names of the provinces. You do this at every debate. You say, no, don't interrupt me. I didn't interrupt you."} -``` diff --git a/_archive/whisper-torchserve/config.yaml b/_archive/whisper-torchserve/config.yaml deleted file mode 100644 index c3ff12df3..000000000 --- a/_archive/whisper-torchserve/config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -environment_variables: {} -external_package_dirs: [] -model_metadata: {} -model_name: Whisper Torchserve -python_version: py310 -requirements: - - torch==2.1.0 - - torchserve==0.9.0 - - ffmpeg-python==0.2.0 - - transformers==4.37.2 - - nvgpu==0.10.0 - - httpx==0.27.0 -resources: - accelerator: T4 - use_gpu: true -model_cache: - - repo_id: htrivedi99/whisper-torchserve - use_volume: false -secrets: {} -system_packages: - - ffmpeg - - openjdk-11-jdk -runtime: - predict_concurrency: 128 diff --git a/_archive/whisper-torchserve/data/config.properties b/_archive/whisper-torchserve/data/config.properties deleted file mode 100644 index 29f59e4b2..000000000 --- a/_archive/whisper-torchserve/data/config.properties +++ /dev/null @@ -1,21 +0,0 @@ -inference_address=http://0.0.0.0:8888 -batch_size=16 -ipex_enable=true -async_logging=true - -models={\ - "whisper_base": {\ - "1.0": {\ - "defaultVersion": true,\ - "marName": "whisper_base.mar",\ - "minWorkers": 1,\ - "maxWorkers": 4,\ - "batchSize": 16,\ - "maxBatchDelay": 250,\ - "responseTimeout": 120\ - }\ - }\ -} - -# maxBatchDelay is the amount of time to wait for the batch size to fill up. Default is 250 ms. -# default_workers_per_model=2 diff --git a/_archive/whisper-torchserve/model/__init__.py b/_archive/whisper-torchserve/model/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/_archive/whisper-torchserve/model/model.py b/_archive/whisper-torchserve/model/model.py deleted file mode 100644 index 4789c5116..000000000 --- a/_archive/whisper-torchserve/model/model.py +++ /dev/null @@ -1,69 +0,0 @@ -import base64 -import multiprocessing -import os -import subprocess -from typing import Dict - -import httpx -import requests -from huggingface_hub import snapshot_download - -TORCHSERVE_ENDPOINT = "http://0.0.0.0:8888/predictions/whisper_base" -TORCHSERVE_HEALTH_ENDPOINT = "http://0.0.0.0:8888/ping" - - -class Model: - def __init__(self, **kwargs): - self._data_dir = kwargs["data_dir"] - self._model = None - self.torchserver_ready = False - - def start_torchserver(self): - subprocess.run( - [ - "torchserve", - "--start", - "--model-store", - f"{self._data_dir}/model_store", - "--models", - "whisper_base.mar", - "--foreground", - "--no-config-snapshots", - "--ts-config", - f"{self._data_dir}/config.properties", - ], - check=True, - ) - - def load(self): - snapshot_download( - "htrivedi99/whisper-torchserve", - local_dir=os.path.join(self._data_dir, "model_store"), - max_workers=4, - ) - logging.info("⚑️ Weights Downloaded Successfully!") - - process = multiprocessing.Process(target=self.start_torchserver) - process.start() - - # Need to wait for the torchserve server to start up - while not self.torchserver_ready: - try: - res = requests.get(TORCHSERVE_HEALTH_ENDPOINT) - if res.status_code == 200: - self.torchserver_ready = True - logging.info("πŸ”₯Torchserve is ready!") - except Exception: - logging.info("⏳Torchserve is loading...") - time.sleep(5) - - async def predict(self, request: Dict): - audio_base64 = request.get("audio") - audio_bytes = base64.b64decode(audio_base64) - - async with httpx.AsyncClient() as client: - res = await client.post( - TORCHSERVE_ENDPOINT, files={"data": (None, audio_bytes)}, timeout=120 - ) - transcription = res.text - return {"output": transcription} From 5543cc0a58aa525b10ba9eb4f3edd14a96df4929 Mon Sep 17 00:00:00 2001 From: Pat Date: Thu, 12 Feb 2026 18:30:28 -0600 Subject: [PATCH 4/4] Replace manual ci.yaml with auto-discovery for CI matrix The 348-line ci.yaml required manual updates for every new example, causing CI failures when entries were missed. CI now auto-discovers examples by walking for config.yaml directories, with an opt-out ci_excludes.yaml for exceptions. Also hardens the GitHub Actions workflow per best practices: - Pin all actions to SHA commits instead of floating tags - Add top-level permissions (contents: read) - Add timeout-minutes to all jobs - Add pip caching to setup-python steps --- .claude/CLAUDE.md | 2 +- .github/workflows/test-examples.yml | 40 ++-- CONTRIBUTING.md | 1 - _internal/README.md | 10 +- _internal/bin/discover_examples.py | 65 ++++++ _internal/bin/test_all.py | 101 ++++---- _internal/bin/validate_ci.py | 8 - ci.yaml | 347 ---------------------------- ci_excludes.yaml | 4 + 9 files changed, 146 insertions(+), 432 deletions(-) create mode 100644 _internal/bin/discover_examples.py delete mode 100644 _internal/bin/validate_ci.py delete mode 100644 ci.yaml create mode 100644 ci_excludes.yaml diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index d33dbe517..661eea3f8 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -105,7 +105,7 @@ Reference example: `image/stable-diffusion/stable-diffusion-xl-1.0` ## CI -Active examples should be listed in `ci.yaml` at the repo root. The test suite (`_internal/bin/test_all.py`) validates all configs, READMEs, naming, links, pinning, and CI completeness. +CI auto-discovers examples by finding directories with `config.yaml`, skipping `_archive/` and `_internal/`. To exclude a specific example from CI, add its path to `ci_excludes.yaml` at the repo root. The test suite (`_internal/bin/test_all.py`) validates all configs, READMEs, naming, links, and pinning. Run tests locally: ```bash diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index ddabb5436..e223057ee 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -6,15 +6,20 @@ on: - cron: '0 6 * * *' workflow_dispatch: +permissions: + contents: read + jobs: validate: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python environment - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.11.4' + cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip @@ -25,21 +30,25 @@ jobs: generate_tests: runs-on: ubuntu-latest + timeout-minutes: 5 outputs: tests: ${{ steps.generate-matrix.outputs.tests }} steps: - - uses: actions/checkout@v4 - - name: yq - portable yaml processor - uses: mikefarah/yq@v4.35.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.11.4' + - run: pip install pyyaml - name: Generate all tests that need to be run id: generate-matrix run: | - TESTS=$(cat ci.yaml | yq e ".tests" | yq eval -o=json | jq -c .) + TESTS=$(python _internal/bin/discover_examples.py) echo "tests=$TESTS" >> $GITHUB_OUTPUT ci: runs-on: ubuntu-latest + timeout-minutes: 30 needs: - validate - generate_tests @@ -49,28 +58,29 @@ jobs: matrix: test: ${{ fromJSON(needs.generate_tests.outputs.tests) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python environment - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.11.4' - - name: Install dependencies (if any) + cache: 'pip' + - name: Install dependencies run: | python -m pip install --upgrade pip pip install git+https://github.com/basetenlabs/truss.git requests tenacity --upgrade - - name: Test => (${{ matrix.test }} + - name: Test ${{ matrix.test }} run: | - python _internal/bin/test_example.py ${{secrets.BASETEN_API_KEY}} ${{matrix.test}} + python _internal/bin/test_example.py ${{ secrets.BASETEN_API_KEY }} ${{ matrix.test }} + report_to_slack: runs-on: ubuntu-latest + timeout-minutes: 5 if: always() && github.ref == 'refs/heads/main' needs: - ci steps: - - name: get-branch - run: echo ${{ github.ref }} - - name: show-slack-status - uses: 8398a7/action-slack@v3 + - name: Report CI result + uses: 8398a7/action-slack@77eaa4f1c608a7d68b38af4e3f739dcd8cba273e # v3.19.0 with: status: custom fields: author, job, commit, repo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab45a3ce9..3efbf29de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,6 @@ We welcome new models and improvements to existing examples. Please open a PR. 2. Include a `config.yaml` with `model_name`, `description`, and `example_model_input`. 3. Include a `README.md` with deploy and invoke instructions. 4. Pin all Python requirements to specific versions. -5. Add your example path to `ci.yaml` at the repo root. ## Validate locally diff --git a/_internal/README.md b/_internal/README.md index 0d0d4b0fe..ec6a22ebf 100644 --- a/_internal/README.md +++ b/_internal/README.md @@ -26,7 +26,7 @@ _internal/ |--------|-------------| | `test_all.py` | Comprehensive local test suite: config validation, README checks, naming conventions, link validation, CI completeness | | `validate_all.py` | Walks every config.yaml and produces a structured validation report (markdown, JSON, or CSV) | -| `validate_ci.py` | Loads every path listed in the top-level `ci.yaml` with `truss.load()` to verify CI paths are valid | +| `discover_examples.py` | Auto-discovers all testable example directories and outputs a JSON array; used by CI to generate the test matrix | | `test_example.py` | CI script: detects changed model from git diff, pushes to staging, runs inference, cleans up old deployments | | `test_truss_deploy.py` | Deploys a truss to staging via `truss push`, invokes inference with `example_model_input`, deactivates after | | `generate_readmes.py` | Auto-generates standardized README.md files for every non-archived example from config.yaml metadata | @@ -73,10 +73,10 @@ The test suite runs 11 checks: 5. README link/path validation 6. `example_model_input` format validation 7. Requirements version pinning -8. CI path validation against `ci.yaml` +8. CI excludes validation against `ci_excludes.yaml` 9. TRT-LLM OpenAI-compatible tag presence 10. `model.py` syntax validation (AST parse) -11. CI completeness (every example appears in `ci.yaml`) +11. Discovery sanity check (auto-discovery finds >100 examples) ### Other validation commands @@ -86,8 +86,8 @@ python _internal/bin/validate_all.py python _internal/bin/validate_all.py --json python _internal/bin/validate_all.py --csv -# Quick CI path check -python _internal/bin/validate_ci.py +# List all auto-discovered examples +python _internal/bin/discover_examples.py --pretty ``` ### README generation diff --git a/_internal/bin/discover_examples.py b/_internal/bin/discover_examples.py new file mode 100644 index 000000000..a4dcdad5f --- /dev/null +++ b/_internal/bin/discover_examples.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Auto-discover all testable example directories in the repository. + +Walks the repo looking for directories containing a config.yaml, +skipping internal/archived paths and any paths listed in ci_excludes.yaml. + +Outputs a sorted JSON array of relative paths to stdout. +Used by the GitHub Actions workflow to generate the CI matrix. + +Usage: + python _internal/bin/discover_examples.py # compact JSON + python _internal/bin/discover_examples.py --pretty # indented JSON +""" + +import json +import os +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +SKIP_DIRS = {".git", ".github", "__pycache__", "node_modules", ".venv", "venv"} + +SKIP_PREFIXES = ("_archive", "_internal") + + +def load_excludes(repo_root: Path) -> set[str]: + """Load excluded paths from ci_excludes.yaml if it exists.""" + excludes_path = repo_root / "ci_excludes.yaml" + if not excludes_path.exists(): + return set() + with open(excludes_path) as f: + data = yaml.safe_load(f) + if not data or not isinstance(data, dict): + return set() + return set(data.get("exclude", []) or []) + + +def discover_examples(repo_root: Path) -> list[str]: + """Find all directories containing a config.yaml, minus excludes.""" + excludes = load_excludes(repo_root) + examples = [] + for dirpath, dirnames, filenames in os.walk(repo_root): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + if "config.yaml" in filenames: + rel = str(Path(dirpath).relative_to(repo_root)) + if any(rel.startswith(prefix) for prefix in SKIP_PREFIXES): + continue + if rel in excludes: + continue + examples.append(rel) + return sorted(examples) + + +def main(): + pretty = "--pretty" in sys.argv + examples = discover_examples(REPO_ROOT) + indent = 2 if pretty else None + print(json.dumps(examples, indent=indent)) + + +if __name__ == "__main__": + main() diff --git a/_internal/bin/test_all.py b/_internal/bin/test_all.py index 4789acc15..5a41a628b 100644 --- a/_internal/bin/test_all.py +++ b/_internal/bin/test_all.py @@ -9,7 +9,7 @@ 5. Link/path validation (README links point to real dirs) 6. example_model_input format validation 7. Requirements pinning check - 8. CI path validation + 8. CI excludes validation Usage: python _internal/bin/test_all.py @@ -547,35 +547,35 @@ def test_requirements_pinning(): print(f" {unpinned_count} configs have unpinned requirements") -# ─── Test 8: CI path validation ────────────────────────────────────────────── +# ─── Test 8: CI excludes validation ────────────────────────────────────────── -def test_ci_paths(): - print("\n== Test 8: CI Path Validation ==") - ci_path = REPO_ROOT / "ci.yaml" - if not ci_path.exists(): - log_fail("ci.yaml not found") +def test_ci_excludes(): + print("\n== Test 8: CI Excludes Validation ==") + excludes_path = REPO_ROOT / "ci_excludes.yaml" + if not excludes_path.exists(): + log_pass("ci_excludes.yaml not found (no excludes, OK)") return - with open(ci_path) as f: - ci = yaml.safe_load(f) - - tests = ci.get("tests", []) - print(f" {len(tests)} CI test paths") - - for test_path in tests: - full_path = REPO_ROOT / test_path - if full_path.exists(): - # Also verify it loads - try: - truss.load(str(full_path)) - log_pass(f"ci.yaml: {test_path} exists and loads") - except Exception as e: - log_fail( - f"ci.yaml: {test_path} exists but truss.load() fails: {str(e)[:100]}" - ) + with open(excludes_path) as f: + data = yaml.safe_load(f) + + if not data or not isinstance(data, dict): + log_pass("ci_excludes.yaml is empty (OK)") + return + + excludes = data.get("exclude", []) or [] + print(f" {len(excludes)} excluded paths") + + for path in excludes: + full_path = REPO_ROOT / path + config_path = full_path / "config.yaml" + if not full_path.exists(): + log_fail(f"ci_excludes.yaml: {path} does not exist") + elif not config_path.exists(): + log_fail(f"ci_excludes.yaml: {path} has no config.yaml") else: - log_fail(f"ci.yaml: {test_path} does not exist") + log_pass(f"ci_excludes.yaml: {path} is a valid example") # ─── Test 9: TRT-LLM openai-compatible tag ────────────────────────────────── @@ -643,38 +643,29 @@ def test_model_py_syntax(): print(f" Checked {checked} model.py files") -# ─── Test 10: CI completeness ─────────────────────────────────────────────── - - -def test_ci_completeness(): - print("\n== Test 11: CI Completeness (ci.yaml covers all examples) ==") - ci_path = REPO_ROOT / "ci.yaml" - if not ci_path.exists(): - log_fail("ci.yaml not found") - return +# ─── Test 11: Discovery sanity ────────────────────────────────────────────── - with open(ci_path) as f: - ci = yaml.safe_load(f) - ci_paths = set(ci.get("tests", [])) +def test_discovery_sanity(): + print("\n== Test 11: Discovery Sanity Check ==") configs = find_all_configs(REPO_ROOT) - missing_from_ci = [] - - for config_dir in configs: - rel = config_dir.relative_to(REPO_ROOT) - rel_str = str(rel) - if rel_str.startswith("_internal") or "_archive" in rel.parts: - continue - if rel_str not in ci_paths: - missing_from_ci.append(rel_str) - log_fail(f"ci.yaml missing: {rel_str}") - else: - log_pass(f"ci.yaml covers: {rel_str}") - - if missing_from_ci: - print(f" {len(missing_from_ci)} examples not in ci.yaml") + # Filter same way as discovery script + examples = [ + str(c.relative_to(REPO_ROOT)) + for c in configs + if not str(c.relative_to(REPO_ROOT)).startswith("_internal") + and "_archive" not in c.relative_to(REPO_ROOT).parts + ] + count = len(examples) + print(f" Auto-discovery found {count} examples") + + if count > 100: + log_pass(f"Discovery found {count} examples (> 100 threshold)") else: - print(f" ci.yaml covers all {len(ci_paths)} examples") + log_fail( + f"Discovery found only {count} examples (expected > 100) β€” " + "discovery logic may be broken" + ) # ─── Main ──────────────────────────────────────────────────────────────────── @@ -692,10 +683,10 @@ def main(): test_readme_links() test_example_model_input() test_requirements_pinning() - test_ci_paths() + test_ci_excludes() test_trt_llm_tags() test_model_py_syntax() - test_ci_completeness() + test_discovery_sanity() print("\n" + "=" * 60) print(f" Results: {passed} passed, {failed} failed, {warnings} warnings") diff --git a/_internal/bin/validate_ci.py b/_internal/bin/validate_ci.py deleted file mode 100644 index d62361221..000000000 --- a/_internal/bin/validate_ci.py +++ /dev/null @@ -1,8 +0,0 @@ -import truss -import yaml - -with open("ci.yaml", "r") as file: - paths = yaml.safe_load(file) - -for path in paths["tests"]: - _ = truss.load(path) diff --git a/ci.yaml b/ci.yaml deleted file mode 100644 index cdedf0eb8..000000000 --- a/ci.yaml +++ /dev/null @@ -1,347 +0,0 @@ -# Canonical list of all examples in the repository. -# test_all.py (Test 8) validates that every path here exists and loads with truss.load(). -tests: - # ── tutorials ────────────────────────────────────────────────────────────── - - tutorials/getting-started-bert - - tutorials/llm-basics - - tutorials/llm-streaming - - tutorials/image-generation - - tutorials/speech-to-text - - tutorials/cached-weights - - tutorials/system-packages - - tutorials/dynamic-batching - - tutorials/private-huggingface - - # ── llm ──────────────────────────────────────────────────────────────────── - # llm/cogito - - llm/cogito/cogito-v2-preview-deepseek-671B-MoE-vllm - - llm/cogito/cogito-v2-preview-llama-109B-MoE-vllm - - llm/cogito/cogito-v2-preview-llama-405B-vllm - - llm/cogito/cogito-v2-preview-llama-70B-vllm - # llm/cogvlm - - llm/cogvlm - # llm/deepseek - - llm/deepseek/deepseek-ocr - - llm/deepseek/deepseek-vl2 - - llm/deepseek/engine-deepseek-r1-distill-llama-70b - - llm/deepseek/engine-deepseek-r1-distill-llama-8b - - llm/deepseek/engine-deepseek-r1-distill-qwen-14b - - llm/deepseek/engine-deepseek-r1-distill-qwen-32b - - llm/deepseek/engine-deepseek-r1-distill-qwen-7b - # llm/falcon - - llm/falcon/falcon3-3B-trt-llm-engine-high-throughput - # llm/gemma - - llm/gemma/gemma-2-27b-it-vllm - - llm/gemma/gemma-2-9b-it-vllm - - llm/gemma/gemma-3-27b-it - # llm/llama - - llm/llama/engine-llama-3-1-70b-instruct - - llm/llama/engine-llama-3-1-8b-instruct - - llm/llama/engine-llama-3-3-70b-instruct - - llm/llama/engine-llama-3.1-405b-instruct - - llm/llama/llama-3-1-405b-instruct - - llm/llama/llama-3-1-70b-instruct - - llm/llama/llama-3-1-8b-instruct - - llm/llama/llama-3-1-8b-instruct-sglang - - llm/llama/llama-3-2-11b-vision-instruct - - llm/llama/llama-3-70b-instruct - - llm/llama/llama-3-8b-instruct - - llm/llama/llama-4-maverick-17b-128e-instruct-fp8-vllm - - llm/llama/llama-4-scout-17b-16e-instruct-bf16-vllm - - llm/llama/tinyllama-1.1B-chat-v1.0 - # llm/llava - - llm/llava/llava-1.6-sgl - - llm/llava/llava-v1.5-7b - - llm/llava/llava-v1.6-34b - # llm/lora - - llm/lora/engine-lora - - llm/lora/sglang-lora - - llm/lora/vllm-lora - # llm/midnight - - llm/midnight - # llm/minimax - - llm/minimax/minimax-m2-1 - # llm/mistral - - llm/mistral/engine-devstral - - llm/mistral/engine-mistral-small-3 - - llm/mistral/engine-mixtral-8x22b-instruct - - llm/mistral/engine-mixtral-8x7b-instruct - - llm/mistral/mistral-7b - - llm/mistral/mistral-small-3.1 - - llm/mistral/mixtral-8x22b - - llm/mistral/mixtral-8x22b-trt-int8-weights-only - - llm/mistral/mixtral-8x7b-instruct-trt-llm - - llm/mistral/mixtral-8x7b-instruct-trt-llm-h100 - - llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant - - llm/mistral/mixtral-8x7b-instruct-trt-llm-weights-only-quant-h100 - - llm/mistral/mixtral-8x7b-instruct-vllm - - llm/mistral/mixtral-8x7b-instruct-vllm-a100-t-tp2 - - llm/mistral/pixtral-12b - # llm/nemotron - - llm/nemotron/llama-3-1-nemotron-70b-instruct - - llm/nemotron/llama-3-1-nemotron-nano-vl-8b-v1 - - llm/nemotron/llama-nemoretriever-colembed-3b-v1 - - llm/nemotron/nemotron-3-nano - - llm/nemotron/nemotron-3-nano-nvfp4 - - llm/nemotron/nemotron-nano-12b-v2-vl-bf16 - - llm/nemotron/nemotron-ultra-253b - # llm/nsql - - llm/nsql - # llm/openai - - llm/openai/gpt-oss-120b - - llm/openai/gpt-oss-20b - # llm/personaplex - - llm/personaplex-7b-v1 - # llm/phi - - llm/phi/phi-3-mini-128k-instruct - - llm/phi/phi-3-mini-4k-instruct - - llm/phi/phi-3.5-mini - # llm/qwen - - llm/qwen/engine-qwen-2-5-14b-coder-instruct - - llm/qwen/engine-qwen-2-5-14b-instruct - - llm/qwen/engine-qwen-2-5-32b-coder-instruct - - llm/qwen/engine-qwen-2-5-32b-instruct - - llm/qwen/engine-qwen-2-5-3b-instruct - - llm/qwen/engine-qwen-2-5-72b-instruct - - llm/qwen/engine-qwen-2-5-72b-math-instruct - - llm/qwen/engine-qwen-2-5-7b-coder-instruct - - llm/qwen/engine-qwen-2-5-7b-instruct - - llm/qwen/engine-qwen-2-5-7b-math-instruct - - llm/qwen/engine-qwen-3-06b - - llm/qwen/engine-qwen-3-30b-a3b-instruct-2507 - - llm/qwen/engine-qwen-3-32b - - llm/qwen/engine-qwen-3-4b - - llm/qwen/qwen-3-235B-A22B-instruct-2507-trt - - llm/qwen/qwen-3-235B-sglang - - llm/qwen/qwen-3-30B-A3-coder - - llm/qwen/qwen-3-30B-A3-sglang - - llm/qwen/qwen-3-30B-A3-vllm - - llm/qwen/qwen-3-32B-sglang - - llm/qwen/qwen-3-next-80B-A3-instruct-sglang - - llm/qwen/qwen-3-next-80B-A3-thinking-sglang - - llm/qwen/qwen-3-vl-30b-a3b-instruct - - llm/qwen/qwen-3-vl-30b-a3b-thinking - - llm/qwen/qwen-3-vl-32b - - llm/qwen/qwen-coder-next - - llm/qwen/qwen-image - - llm/qwen/qwen-vl - # llm/seed - - llm/seed/seed-llm - # llm/z-ai - - llm/z-ai/glm-4-5-air-fp8 - - llm/z-ai/glm-4-5-fp8 - - llm/z-ai/glm-4-5-v - - llm/z-ai/glm-4-6-fp8 - - llm/z-ai/glm-4-7-flash - - # ── embeddings ───────────────────────────────────────────────────────────── - # embeddings/bei - - embeddings/bei/alibaba-nlp-gte-modernbert-base-embedding - - embeddings/bei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding - - embeddings/bei/alibaba-nlp-gte-qwen2-7b-instruct-embedding - - embeddings/bei/alibaba-nlp-gte-reranker-modernbert-base - - embeddings/bei/allenai-llama-3.1-tulu-3-8b-reward-model-fp8 - - embeddings/bei/baai-bge-en-icl-embedding-fp8 - - embeddings/bei/baai-bge-large-en-v1.5-embedding - - embeddings/bei/baai-bge-m3-embedding-dense - - embeddings/bei/baai-bge-multilingual-gemma2-multilingual-embedding - - embeddings/bei/baai-bge-reranker-large - - embeddings/bei/baai-bge-reranker-large/BEI-Bert-baai-bge-reranker-large - - embeddings/bei/baai-bge-reranker-v2-m3-multilingual - - embeddings/bei/baseten-example-meta-llama-3-70b-instructforsequenceclassification-fp8 - - embeddings/bei/codefuse-ai-f2llm-4b-embedding-fp8 - - embeddings/bei/google-embeddinggemma-300m - - embeddings/bei/intfloat-e5-mistral-7b-instruct-embedding-fp8 - - embeddings/bei/intfloat-multilingual-e5-large-instruct - - embeddings/bei/jina-ai-jina-embeddings-v2-base-en - - embeddings/bei/jinaai-jina-code-embeddings-0.5b-fp8 - - embeddings/bei/jinaai-jina-embeddings-v2-base-code - - embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding - - embeddings/bei/mixedbread-ai-mxbai-embed-large-v1-embedding/BEI-mixedbread-ai-mxbai-embed-large-v1-embedding - - embeddings/bei/mixedbread-ai-mxbai-rerank-base-v2-reranker-fp8 - - embeddings/bei/mixedbread-ai-mxbai-rerank-large-v2-reranker-fp8 - - embeddings/bei/ncbi-medcpt-cross-encoder-reranker - - embeddings/bei/ner-bert-base-ner-uncased - - embeddings/bei/nomic-ai-nomic-embed-code-fp8 - - embeddings/bei/nomic-ai-nomic-embed-text-v1.5 - - embeddings/bei/nomic-ai-nomic-embed-text-v2-moe - - embeddings/bei/nvidia-llama-embed-nemotron-8b - - embeddings/bei/nvidia-llama-nemotron-embed-1b-v2 - - embeddings/bei/papluca-xlm-roberta-base-language-detection-classification - - embeddings/bei/qwen-qwen3-embedding-0.6b-fp8 - - embeddings/bei/qwen-qwen3-embedding-4b-fp4 - - embeddings/bei/qwen-qwen3-embedding-4b-fp8 - - embeddings/bei/qwen-qwen3-embedding-8b-fp8 - - embeddings/bei/qwen-qwen3-reranker-0.6b-fp8 - - embeddings/bei/qwen-qwen3-reranker-4b-fp8 - - embeddings/bei/qwen-qwen3-reranker-8b-fp4 - - embeddings/bei/qwen-qwen3-reranker-8b-fp8 - - embeddings/bei/redis-langcache-embed-v2 - - embeddings/bei/salesforce-sfr-embedding-mistral-fp8 - - embeddings/bei/samlowe-roberta-base-go_emotions-classification - - embeddings/bei/sentence-transformers-all-minilm-l6-v2-embedding - - embeddings/bei/skywork-skywork-reward-llama-3.1-8b-v0.2-reward-model-fp8 - - embeddings/bei/snowflake-snowflake-arctic-embed-l-v2.0 - - embeddings/bei/tanaos-tanaos-ner-v1 - - embeddings/bei/taylorai-bge-micro-v2 - - embeddings/bei/voyageai-voyage-4-nano - - embeddings/bei/whereisai-uae-large-v1-embedding - # embeddings/tei - - embeddings/tei/alibaba-nlp-gte-modernbert-base-embedding - - embeddings/tei/alibaba-nlp-gte-qwen2-1.5b-instruct-embedding - - embeddings/tei/alibaba-nlp-gte-qwen2-7b-instruct-embedding - - embeddings/tei/alibaba-nlp-gte-reranker-modernbert-base - - embeddings/tei/baai-bge-reranker-large - - embeddings/tei/google-embeddinggemma-300m - - embeddings/tei/intfloat-multilingual-e5-large-instruct - - embeddings/tei/jina-ai-jina-embeddings-v2-base-en - - embeddings/tei/jinaai-jina-embeddings-v2-base-code - - embeddings/tei/mixedbread-ai-mxbai-embed-large-v1-embedding - - embeddings/tei/nomic-ai-nomic-embed-text-v1.5 - - embeddings/tei/nomic-ai-nomic-embed-text-v2-moe - - embeddings/tei/redis-langcache-embed-v2 - - embeddings/tei/sentence-transformers-all-minilm-l6-v2-embedding - - embeddings/tei/taylorai-bge-micro-v2 - # embeddings/other - - embeddings/clip - - embeddings/text-embeddings-inference - - # ── image ────────────────────────────────────────────────────────────────── - - image/comfyui - - image/comfyui/examples/anime-style-transfer - - image/control-net-qrcode - - image/deepfloyd-xl - - image/dis-segmentation - - image/flux-dev-trt-b200 - - image/flux/dev - - image/flux/schnell - - image/fotographer/zenctrl - - image/gfp-gan - - image/image-segmentation - - image/ip-adapter - - image/magic-animate - - image/playground-v2-aesthetic - - image/sana/sana-1600m - - image/sana/sana-600m - - image/segment-anything - - image/stable-diffusion/dreamshaper-lcm - - image/stable-diffusion/playground-v2-trt - - image/stable-diffusion/sd-textual-inversion - - image/stable-diffusion/sd-turbo - - image/stable-diffusion/sdxl-controlnet - - image/stable-diffusion/sdxl-controlnet-canny - - image/stable-diffusion/sdxl-controlnet-depth - - image/stable-diffusion/sdxl-lightning - - image/stable-diffusion/sdxl-lora - - image/stable-diffusion/sdxl-lora-swapping - - image/stable-diffusion/sdxl-turbo - - image/stable-diffusion/stable-diffusion - - image/stable-diffusion/stable-diffusion-3-medium - - image/stable-diffusion/stable-diffusion-inpainting-trt - - image/stable-diffusion/stable-diffusion-xl-1.0 - - image/stable-diffusion/stable-diffusion-xl-1.0-trt - - image/stable-diffusion/stable-diffusion-xl-1.0-trt-h100 - - image/stable-diffusion/stable-video-diffusion - - # ── audio ────────────────────────────────────────────────────────────────── - - audio/audiogen-medium - - audio/chatterbox-tts - - audio/kokoro - - audio/metavoice-1b - - audio/musicgen-large - - audio/musicgen-melody - - audio/nvidia-parakeet - - audio/orpheus-3b-websockets - - audio/orpheus-best-performance - - audio/piper-tts - - audio/qwen-asr - - audio/qwen-omni - - audio/qwen-omni-thinker - - audio/sesame-csm-1b - - audio/ultravox - - audio/voxtral-streaming-4b - - audio/whisper/faster-whisper-small - - audio/whisper/faster-whisper-v2 - - audio/whisper/faster-whisper-v3 - - audio/whisper/whisper-openai - - audio/whisper/whisper-streaming - - audio/whisper/whisper-v3 - - audio/whisper/whisper-v3-base64 - - audio/whisper/whisperx - - audio/xtts-streaming - - audio/xtts-v2 - - # ── optimized ────────────────────────────────────────────────────────────── - # optimized/bisv2 - - optimized/bisv2/deepseek-ai-deepseek-r1-distill-llama-70b-fp4 - - optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp4-mlp-only - - optimized/bisv2/meta-llama-llama-3.2-3b-instruct-fp8 - - optimized/bisv2/meta-llama-llama-3.3-70b-instruct-fp4 - - optimized/bisv2/nvidia-llama-3.1-8b-instruct-fp4 - - optimized/bisv2/nvidia-qwen3-30b-a3b-fp4 - - optimized/bisv2/nvidia-qwen3-8b-fp4 - - optimized/bisv2/qwen-qwen2.5-coder-7b-instruct - - optimized/bisv2/qwen-qwen2.5-coder-7b-instruct-fp4 - - optimized/bisv2/qwen-qwen3-32b-fp4 - - optimized/bisv2/qwen-qwen3-4b-fp8 - # optimized/briton - - optimized/briton/deepseek-ai-deepseek-r1-distill-llama-70b-fp8 - - optimized/briton/deepseek-ai-deepseek-r1-distill-qwen-32b-fp8 - - optimized/briton/google-gemma-3-1b-it - - optimized/briton/google-gemma-3-270m-it - - optimized/briton/google-gemma-3-27b-it - - optimized/briton/google-gemma-3-27b-it-speculative-lookahead - - optimized/briton/meta-llama-llama-3.1-405b-fp8 - - optimized/briton/meta-llama-llama-3.1-8b-instruct-with-speculative-lookahead-decoding-fp8 - - optimized/briton/meta-llama-llama-3.2-1b-instruct-fp8 - - optimized/briton/meta-llama-llama-3.2-3b-instruct - - optimized/briton/meta-llama-llama-3.2-3b-instruct-calib-dataset-fp8 - - optimized/briton/meta-llama-llama-3.2-3b-instruct-fp8 - - optimized/briton/meta-llama-llama-3.3-70b-instruct-fp4 - - optimized/briton/meta-llama-llama-3.3-70b-instruct-fp8 - - optimized/briton/meta-llama-llama-3.3-70b-instruct-tp4-fp8 - - optimized/briton/microsoft-phi-4-fp8 - - optimized/briton/mistralai-mistral-7b-instruct-v0.3 - - optimized/briton/mistralai-mistral-small-24b-instruct-2501-fp8 - - optimized/briton/qwen-qwen2.5-72b-instruct-tp2-fp8 - - optimized/briton/qwen-qwen2.5-7b-instruct-with-speculative-lookahead-decoding-fp8 - - optimized/briton/qwen-qwen2.5-coder-7b-instruct-calib-dataset-fp4-mlp-only - - optimized/briton/qwen-qwen2.5-coder-7b-instruct-min-latency-fp8 - - optimized/briton/qwen-qwen3-235b-a22b-instruct-2507-fp8 - - optimized/briton/qwen-qwen3-30b-a3b - - optimized/briton/qwen-qwen3-30b-a3b-fp8 - - optimized/briton/qwen-qwen3-30b-a3b-instruct-2507-fp8 - - optimized/briton/qwen-qwen3-32b-fp4 - - optimized/briton/qwen-qwen3-32b-fp4-mlp-only - - optimized/briton/qwen-qwen3-32b-fp8 - - optimized/briton/qwen-qwen3-8b-min-latency-fp8 - - optimized/briton/qwen-qwq-32b-reasoning-fp8 - - optimized/briton/qwen-qwq-32b-reasoning-with-speculative-fp8 - - optimized/briton/tiiuae-falcon3-10b-instruct-fp8 - - # ── infrastructure ───────────────────────────────────────────────────────── - - infrastructure/autodesk-wala - - infrastructure/binocular - - infrastructure/custom-engine-builder-control - - infrastructure/custom-server/deepseek-v2-5-instruct-sglang - - infrastructure/custom-server/infinity-embedding-server - - infrastructure/custom-server/llama3-70b-instruct-lmdeploy - - infrastructure/custom-server/llama3-70b-instruct-sglang - - infrastructure/custom-server/llama3-8b-instruct-lmdeploy - - infrastructure/custom-server/llama3-8b-instruct-sglang - - infrastructure/custom-server/pixtral-12b - - infrastructure/custom-server/ultravox-0.4 - - infrastructure/custom-server/ultravox-0.5-8b - - infrastructure/custom-server/ultravox-0.6-70b - - infrastructure/custom-server/voxtral-mini-3b-2507 - - infrastructure/custom-server/voxtral-small-24b-2507 - - infrastructure/grpc - - infrastructure/jsonformatter - - infrastructure/layoutlm-document-qa - - infrastructure/llama-cpp-server - - infrastructure/metrics/datadog - - infrastructure/model-cache - - infrastructure/multiprocessing - - infrastructure/ngram-speculator/truss - - infrastructure/ngram-speculator/trussless - - infrastructure/paddlepaddle/paddleocr-vl diff --git a/ci_excludes.yaml b/ci_excludes.yaml new file mode 100644 index 000000000..3393c8d6e --- /dev/null +++ b/ci_excludes.yaml @@ -0,0 +1,4 @@ +# Paths to exclude from CI auto-discovery. +# Examples are discovered automatically by finding directories with config.yaml. +# Add paths here to opt specific examples out of CI. +exclude: []