From 222ce23ff42f758bb1486fa37cebb8dc38b43fef Mon Sep 17 00:00:00 2001 From: ha-sante <90225652+ha-sante@users.noreply.github.com> Date: Fri, 3 Nov 2023 11:15:46 +0000 Subject: [PATCH 01/12] VoyageAIEmbeddingFunction - New Embedding Function This adds embedding function support for voyageai.com. Documentation: - https://docs.voyageai.com/tutorials/ --- chromadb/utils/embedding_functions.py | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index aaef53c01e2..c224f32c682 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -479,6 +479,46 @@ def __call__(self, texts: Documents) -> Embeddings: return embeddings +class VoyageAIEmbeddingFunction(EmbeddingFunction): + def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int = 8): + """ + Initialize the VoyageAIEmbeddingFunction. + + Args: + api_key (str): Your API key for the HuggingFace API. + model_name (str, optional): The name of the model to use for text embeddings. Defaults to "voyage-01". + batch_size (int, optional): The number of documents to send at a time. Defaults to 8 (The max supported 3rd Nov 2023). + """ + if batch_size > 8: + print(f"Voyage AI as of (3rd Nov 2023) has a batch size of max 8") + + if not api_key: + raise ValueError("Please provide a VoyageAI API key.") + + try: + import voyageai + from voyageai import get_embeddings, + voyageai.api_key = api_key # add you Voyage API KEY + except ImportError: + raise ValueError("The VoyageAI python package is not installed. Please install it with `pip install voyageai`") + + self.batch_size = batch_size + self.model = model_name + self.get_embeddings = get_embeddings + + def __call__(self, texts: Documents) -> Embeddings: + iters = range(0, len(texts), self.batch_size) + embeddings = [] + for i in iters: + results = self.get_embeddings( + texts[i : i + self.batch_size], + batch_size=self.batch_size, + model=self.model + ) + embeddings += results; + return embeddings; + + # List of all classes in this module _classes = [ name From 574c557ca13ceb1e15652a81c499f34162a8cd02 Mon Sep 17 00:00:00 2001 From: ha-sante <90225652+ha-sante@users.noreply.github.com> Date: Fri, 3 Nov 2023 11:18:57 +0000 Subject: [PATCH 02/12] VoyageAIEmbeddingFunction - New Embedding Function --- chromadb/utils/embedding_functions.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index c224f32c682..eaf8468ceb0 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -507,6 +507,20 @@ def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int self.get_embeddings = get_embeddings def __call__(self, texts: Documents) -> Embeddings: + """ + Get the embeddings for a list of texts. + + Args: + texts (Documents): A list of texts to get embeddings for. + + Returns: + Embeddings: The embeddings for the texts. + + Example: + >>> voyage_ef = VoyageAIEmbeddingFunction(api_key="your_api_key") + >>> texts = ["Hello, world!", "How are you?"] + >>> embeddings = voyage_ef(texts) + """ iters = range(0, len(texts), self.batch_size) embeddings = [] for i in iters: From ff2b11f011574a7bb620a68eb0601c59c6242a00 Mon Sep 17 00:00:00 2001 From: ha-sante <90225652+ha-sante@users.noreply.github.com> Date: Fri, 3 Nov 2023 11:43:48 +0000 Subject: [PATCH 03/12] VoyageAIEmbeddingFunction - New Embedding Function --- chromadb/utils/embedding_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index eaf8468ceb0..5900781f90c 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -497,11 +497,11 @@ def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int try: import voyageai - from voyageai import get_embeddings, - voyageai.api_key = api_key # add you Voyage API KEY + from voyageai import get_embeddings except ImportError: raise ValueError("The VoyageAI python package is not installed. Please install it with `pip install voyageai`") + voyageai.api_key = api_key # Voyage API Key self.batch_size = batch_size self.model = model_name self.get_embeddings = get_embeddings From 0af33fcac68bd56bb57813eb074a9cd889fe9907 Mon Sep 17 00:00:00 2001 From: ha-sante <90225652+ha-sante@users.noreply.github.com> Date: Fri, 3 Nov 2023 11:59:36 +0000 Subject: [PATCH 04/12] VoyageAIEmbeddingFunction - New Embedding Function --- chromadb/utils/embedding_functions.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index 5900781f90c..c31e155f666 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -481,13 +481,13 @@ def __call__(self, texts: Documents) -> Embeddings: class VoyageAIEmbeddingFunction(EmbeddingFunction): def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int = 8): - """ + """ Initialize the VoyageAIEmbeddingFunction. Args: - api_key (str): Your API key for the HuggingFace API. - model_name (str, optional): The name of the model to use for text embeddings. Defaults to "voyage-01". - batch_size (int, optional): The number of documents to send at a time. Defaults to 8 (The max supported 3rd Nov 2023). + api_key (str): Your API key for the HuggingFace API. + model_name (str, optional): The name of the model to use for text embeddings. Defaults to "voyage-01". + batch_size (int, optional): The number of documents to send at a time. Defaults to 8 (The max supported 3rd Nov 2023). """ if batch_size > 8: print(f"Voyage AI as of (3rd Nov 2023) has a batch size of max 8") @@ -507,19 +507,19 @@ def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int self.get_embeddings = get_embeddings def __call__(self, texts: Documents) -> Embeddings: - """ + """ Get the embeddings for a list of texts. Args: - texts (Documents): A list of texts to get embeddings for. + texts (Documents): A list of texts to get embeddings for. Returns: - Embeddings: The embeddings for the texts. + Embeddings: The embeddings for the texts. Example: - >>> voyage_ef = VoyageAIEmbeddingFunction(api_key="your_api_key") - >>> texts = ["Hello, world!", "How are you?"] - >>> embeddings = voyage_ef(texts) + >>> voyage_ef = VoyageAIEmbeddingFunction(api_key="your_api_key") + >>> texts = ["Hello, world!", "How are you?"] + >>> embeddings = voyage_ef(texts) """ iters = range(0, len(texts), self.batch_size) embeddings = [] @@ -532,7 +532,6 @@ def __call__(self, texts: Documents) -> Embeddings: embeddings += results; return embeddings; - # List of all classes in this module _classes = [ name From 62137852eb201a789f67500fe6d797e7b95779a5 Mon Sep 17 00:00:00 2001 From: ha-sante <90225652+ha-sante@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:00:34 +0000 Subject: [PATCH 05/12] New Embedding Function - VoyageAIEmbeddingFunction --- chromadb/utils/embedding_functions.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index c31e155f666..d09982afdb9 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -506,26 +506,26 @@ def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int self.model = model_name self.get_embeddings = get_embeddings - def __call__(self, texts: Documents) -> Embeddings: + def __call__(self, input: Documents) -> Embeddings: """ Get the embeddings for a list of texts. Args: - texts (Documents): A list of texts to get embeddings for. + input (Documents): A list of texts to get embeddings for. Returns: Embeddings: The embeddings for the texts. Example: >>> voyage_ef = VoyageAIEmbeddingFunction(api_key="your_api_key") - >>> texts = ["Hello, world!", "How are you?"] - >>> embeddings = voyage_ef(texts) + >>> input = ["Hello, world!", "How are you?"] + >>> embeddings = voyage_ef(input) """ - iters = range(0, len(texts), self.batch_size) + iters = range(0, len(input), self.batch_size) embeddings = [] for i in iters: results = self.get_embeddings( - texts[i : i + self.batch_size], + input[i : i + self.batch_size], batch_size=self.batch_size, model=self.model ) From fd495e9f12140236c5aa1845af674d45df3cde20 Mon Sep 17 00:00:00 2001 From: ha-sante <90225652+ha-sante@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:25:56 +0000 Subject: [PATCH 06/12] Update embedding_functions.py --- chromadb/utils/embedding_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index d09982afdb9..e70e7b383f2 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -480,6 +480,7 @@ def __call__(self, texts: Documents) -> Embeddings: class VoyageAIEmbeddingFunction(EmbeddingFunction): + """Embedding function for Voyageai.com""" def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int = 8): """ Initialize the VoyageAIEmbeddingFunction. From c0cbbed7f7b38397b61c21777ac3085c6d5d66f9 Mon Sep 17 00:00:00 2001 From: Trayan Azarov Date: Mon, 8 Apr 2024 08:20:59 +0300 Subject: [PATCH 07/12] feat: Updated to latest API - Updated the EF to the latest API - Added a few more options - Tests --- chromadb/test/ef/test_voyageai.py | 105 +++++++++++++++++++++ chromadb/utils/embedding_functions.py | 131 ++++++++++++++------------ 2 files changed, 177 insertions(+), 59 deletions(-) create mode 100644 chromadb/test/ef/test_voyageai.py diff --git a/chromadb/test/ef/test_voyageai.py b/chromadb/test/ef/test_voyageai.py new file mode 100644 index 00000000000..fe3348e6d3f --- /dev/null +++ b/chromadb/test/ef/test_voyageai.py @@ -0,0 +1,105 @@ +import os + +import pytest + +from chromadb.utils.embedding_functions import VoyageAIEmbeddingFunction + + +def test_voyage() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGEAI_API_KEY", "")) + embeddings = ef(["test doc"]) + assert embeddings is not None + assert len(embeddings) == 1 + assert len(embeddings[0]) > 0 + + +def test_voyage_input_type_query() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction( + api_key=os.environ.get("VOYAGEAI_API_KEY", ""), input_type="query" + ) + embeddings = ef(["test doc"]) + assert embeddings is not None + assert len(embeddings) == 1 + assert len(embeddings[0]) > 0 + + +def test_voyage_input_type_document() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction( + api_key=os.environ.get("VOYAGEAI_API_KEY", ""), input_type="document" + ) + embeddings = ef(["test doc"]) + assert embeddings is not None + assert len(embeddings) == 1 + assert len(embeddings[0]) > 0 + + +def test_voyage_model() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction( + api_key=os.environ.get("VOYAGEAI_API_KEY", ""), model_name="voyage-code-2" + ) + embeddings = ef(["def test():\n return 1"]) + assert embeddings is not None + assert len(embeddings) == 1 + assert len(embeddings[0]) > 0 + + +def test_voyage_truncation_default() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGEAI_API_KEY", "")) + embeddings = ef(["this is a test-message" * 10000]) + assert embeddings is not None + assert len(embeddings) == 1 + assert len(embeddings[0]) > 0 + + +def test_voyage_truncation_enabled() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction( + api_key=os.environ.get("VOYAGEAI_API_KEY", ""), truncation=True + ) + embeddings = ef(["this is a test-message" * 10000]) + assert embeddings is not None + assert len(embeddings) == 1 + assert len(embeddings[0]) > 0 + + +def test_voyage_truncation_disabled() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction( + api_key=os.environ.get("VOYAGEAI_API_KEY", ""), truncation=False + ) + with pytest.raises(Exception, match="your batch has too many tokens"): + ef(["this is a test-message" * 10000]) + + +def test_voyage_no_api_key() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + with pytest.raises(ValueError, match="Please provide a VoyageAI API key"): + VoyageAIEmbeddingFunction(api_key=None) # type: ignore + + +def test_voyage_max_batch_size_exceeded_in_init() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + with pytest.raises(ValueError, match="The maximum batch size supported is"): + VoyageAIEmbeddingFunction(api_key="dummy", max_batch_size=99999999) + + +def test_voyage_max_batch_size_exceeded_in_call() -> None: + if "VOYAGEAI_API_KEY" not in os.environ: + pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") + ef = VoyageAIEmbeddingFunction(api_key="dummy", max_batch_size=1) + with pytest.raises(ValueError, match="The maximum batch size supported is"): + ef(["test doc"] * 2) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index 41becc6e3fa..d2297dc5cf3 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -743,9 +743,7 @@ def __call__(self, input: Union[Documents, Images]) -> Embeddings: class RoboflowEmbeddingFunction(EmbeddingFunction[Union[Documents, Images]]): - def __init__( - self, api_key: str = "", api_url = "https://infer.roboflow.com" - ) -> None: + def __init__(self, api_key: str = "", api_url="https://infer.roboflow.com") -> None: """ Create a RoboflowEmbeddingFunction. @@ -757,7 +755,7 @@ def __init__( api_key = os.environ.get("ROBOFLOW_API_KEY") self._api_url = api_url - self._api_key = api_key + self._api_key = api_key try: self._PILImage = importlib.import_module("PIL.Image") @@ -789,10 +787,10 @@ def __call__(self, input: Union[Documents, Images]) -> Embeddings: json=infer_clip_payload, ) - result = res.json()['embeddings'] + result = res.json()["embeddings"] embeddings.append(result[0]) - + elif is_document(item): infer_clip_payload = { "text": input, @@ -803,13 +801,13 @@ def __call__(self, input: Union[Documents, Images]) -> Embeddings: json=infer_clip_payload, ) - result = res.json()['embeddings'] + result = res.json()["embeddings"] embeddings.append(result[0]) return embeddings - + class AmazonBedrockEmbeddingFunction(EmbeddingFunction[Documents]): def __init__( self, @@ -899,55 +897,70 @@ def __call__(self, input: Documents) -> Embeddings: Embeddings, self._session.post(self._api_url, json={"inputs": input}).json() ) + class VoyageAIEmbeddingFunction(EmbeddingFunction): - """Embedding function for Voyageai.com""" - def __init__(self, api_key: str, model_name: str = "voyage-01", batch_size: int = 8): - """ - Initialize the VoyageAIEmbeddingFunction. - Args: - api_key (str): Your API key for the HuggingFace API. - model_name (str, optional): The name of the model to use for text embeddings. Defaults to "voyage-01". - batch_size (int, optional): The number of documents to send at a time. Defaults to 8 (The max supported 3rd Nov 2023). - """ - if batch_size > 8: - print(f"Voyage AI as of (3rd Nov 2023) has a batch size of max 8") - - if not api_key: - raise ValueError("Please provide a VoyageAI API key.") - - try: - import voyageai - from voyageai import get_embeddings - except ImportError: - raise ValueError("The VoyageAI python package is not installed. Please install it with `pip install voyageai`") - - voyageai.api_key = api_key # Voyage API Key - self.batch_size = batch_size - self.model = model_name - self.get_embeddings = get_embeddings - - def __call__(self, input: Documents) -> Embeddings: - """ - Get the embeddings for a list of texts. - Args: - input (Documents): A list of texts to get embeddings for. - Returns: - Embeddings: The embeddings for the texts. - Example: - >>> voyage_ef = VoyageAIEmbeddingFunction(api_key="your_api_key") - >>> input = ["Hello, world!", "How are you?"] - >>> embeddings = voyage_ef(input) - """ - iters = range(0, len(input), self.batch_size) - embeddings = [] - for i in iters: - results = self.get_embeddings( - input[i : i + self.batch_size], - batch_size=self.batch_size, - model=self.model - ) - embeddings += results; - return embeddings; + """Embedding function for Voyageai.com. API docs - https://docs.voyageai.com/reference/embeddings-api""" + + def __init__( + self, + api_key: str, + model_name: str = "voyage-2", + max_batch_size: int = 128, + truncation: Optional[bool] = True, + input_type: Optional[str] = None, + ): + """ + Initialize the VoyageAIEmbeddingFunction. + Args: + api_key (str): Your API key for the HuggingFace API. + model_name (str, optional): The name of the model to use for text embeddings. Defaults to "voyage-01". + batch_size (int, optional): The number of documents to send at a time. Defaults to 128 (The max supported 7th Apr 2024). see voyageai.VOYAGE_EMBED_BATCH_SIZE for actual max. + truncation (bool, optional): Whether to truncate the input (`True`) or raise an error if the input is too long (`False`). Defaults to `False`. + input_type (str, optional): The type of input text. Can be `None`, `query`, `document`. Defaults to `None`. + """ + + if not api_key: + raise ValueError("Please provide a VoyageAI API key.") + + try: + import voyageai + + if max_batch_size > voyageai.VOYAGE_EMBED_BATCH_SIZE: + raise ValueError( + f"The maximum batch size supported is {voyageai.VOYAGE_EMBED_BATCH_SIZE}." + ) + voyageai.api_key = api_key # Voyage API Key + self._batch_size = max_batch_size + self._model = model_name + self._truncation = truncation + self._client = voyageai.Client() + self._input_type = input_type + except ImportError: + raise ValueError( + "The VoyageAI python package is not installed. Please install it with `pip install voyageai`" + ) + + def __call__(self, input: Documents) -> Embeddings: + """ + Get the embeddings for a list of texts. + Args: + input (Documents): A list of texts to get embeddings for. + Returns: + Embeddings: The embeddings for the texts. + Example: + >>> voyage_ef = VoyageAIEmbeddingFunction(api_key="your_api_key") + >>> input = ["Hello, world!", "How are you?"] + >>> embeddings = voyage_ef(input) + """ + if len(input) > self._batch_size: + raise ValueError(f"The maximum batch size supported is {self._batch_size}.") + results = self._client.embed( + texts=input, + model=self._model, + truncation=self._truncation, + input_type=self._input_type, + ) + return results.embeddings def create_langchain_embedding(langchain_embdding_fn: Any): # type: ignore @@ -1012,7 +1025,7 @@ def __call__(self, input: Documents) -> Embeddings: # type: ignore return ChromaLangchainEmbeddingFunction(embedding_function=langchain_embdding_fn) - + class OllamaEmbeddingFunction(EmbeddingFunction[Documents]): """ This class is used to generate embeddings for a list of texts using the Ollama Embedding API (https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings). @@ -1068,7 +1081,7 @@ def __call__(self, input: Documents) -> Embeddings: ], ) - + # List of all classes in this module _classes = [ name @@ -1078,4 +1091,4 @@ def __call__(self, input: Documents) -> Embeddings: def get_builtins() -> List[str]: - return _classes \ No newline at end of file + return _classes From 2ffd9e3aa067e5dca6790940942b55766f33c89e Mon Sep 17 00:00:00 2001 From: Trayan Azarov Date: Mon, 8 Apr 2024 09:33:31 +0300 Subject: [PATCH 08/12] chore: input_type is now an enum - Updated API key to be VOYAGE_API_KEY - Test cleanup --- chromadb/test/ef/test_voyageai.py | 72 ++++++++++++++++----------- chromadb/utils/embedding_functions.py | 18 ++++--- 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/chromadb/test/ef/test_voyageai.py b/chromadb/test/ef/test_voyageai.py index fe3348e6d3f..ced093192d9 100644 --- a/chromadb/test/ef/test_voyageai.py +++ b/chromadb/test/ef/test_voyageai.py @@ -5,21 +5,33 @@ from chromadb.utils.embedding_functions import VoyageAIEmbeddingFunction +@pytest.fixture(scope="function") +def remove_api_key(): + existing_api_key = None + if "VOYAGE_API_KEY" in os.environ: + existing_api_key = os.environ["VOYAGE_API_KEY"] + print("removing key") + del os.environ["VOYAGE_API_KEY"] + yield + if existing_api_key: + print("setting kye") + os.environ["VOYAGE_API_KEY"] = existing_api_key + + +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") - ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGEAI_API_KEY", "")) + ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGE_API_KEY", "")) embeddings = ef(["test doc"]) assert embeddings is not None assert len(embeddings) == 1 assert len(embeddings[0]) > 0 +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_input_type_query() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") ef = VoyageAIEmbeddingFunction( - api_key=os.environ.get("VOYAGEAI_API_KEY", ""), input_type="query" + api_key=os.environ.get("VOYAGE_API_KEY", ""), + input_type=VoyageAIEmbeddingFunction.InputType.QUERY, ) embeddings = ef(["test doc"]) assert embeddings is not None @@ -27,11 +39,11 @@ def test_voyage_input_type_query() -> None: assert len(embeddings[0]) > 0 +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_input_type_document() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") ef = VoyageAIEmbeddingFunction( - api_key=os.environ.get("VOYAGEAI_API_KEY", ""), input_type="document" + api_key=os.environ.get("VOYAGE_API_KEY", ""), + input_type=VoyageAIEmbeddingFunction.InputType.DOCUMENT, ) embeddings = ef(["test doc"]) assert embeddings is not None @@ -39,11 +51,10 @@ def test_voyage_input_type_document() -> None: assert len(embeddings[0]) > 0 +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_model() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") ef = VoyageAIEmbeddingFunction( - api_key=os.environ.get("VOYAGEAI_API_KEY", ""), model_name="voyage-code-2" + api_key=os.environ.get("VOYAGE_API_KEY", ""), model_name="voyage-01" ) embeddings = ef(["def test():\n return 1"]) assert embeddings is not None @@ -51,21 +62,19 @@ def test_voyage_model() -> None: assert len(embeddings[0]) > 0 +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_truncation_default() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") - ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGEAI_API_KEY", "")) + ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGE_API_KEY", "")) embeddings = ef(["this is a test-message" * 10000]) assert embeddings is not None assert len(embeddings) == 1 assert len(embeddings[0]) > 0 +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_truncation_enabled() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") ef = VoyageAIEmbeddingFunction( - api_key=os.environ.get("VOYAGEAI_API_KEY", ""), truncation=True + api_key=os.environ.get("VOYAGE_API_KEY", ""), truncation=True ) embeddings = ef(["this is a test-message" * 10000]) assert embeddings is not None @@ -73,33 +82,40 @@ def test_voyage_truncation_enabled() -> None: assert len(embeddings[0]) > 0 +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_truncation_disabled() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") ef = VoyageAIEmbeddingFunction( - api_key=os.environ.get("VOYAGEAI_API_KEY", ""), truncation=False + api_key=os.environ.get("VOYAGE_API_KEY", ""), truncation=False ) with pytest.raises(Exception, match="your batch has too many tokens"): ef(["this is a test-message" * 10000]) -def test_voyage_no_api_key() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +def test_voyage_env_api_key() -> None: + VoyageAIEmbeddingFunction() + + +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +def test_voyage_no_api_key(remove_api_key) -> None: + with pytest.raises(ValueError, match="Please provide a VoyageAI API key"): + VoyageAIEmbeddingFunction(api_key=None) # type: ignore + + +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +def test_voyage_no_api_key_in_env(remove_api_key) -> None: with pytest.raises(ValueError, match="Please provide a VoyageAI API key"): VoyageAIEmbeddingFunction(api_key=None) # type: ignore +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_max_batch_size_exceeded_in_init() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") with pytest.raises(ValueError, match="The maximum batch size supported is"): VoyageAIEmbeddingFunction(api_key="dummy", max_batch_size=99999999) +@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") def test_voyage_max_batch_size_exceeded_in_call() -> None: - if "VOYAGEAI_API_KEY" not in os.environ: - pytest.skip("VOYAGEAI_API_KEY not set, not going to test VoyageAI EF.") ef = VoyageAIEmbeddingFunction(api_key="dummy", max_batch_size=1) with pytest.raises(ValueError, match="The maximum batch size supported is"): ef(["test doc"] * 2) diff --git a/chromadb/utils/embedding_functions.py b/chromadb/utils/embedding_functions.py index d2297dc5cf3..76680b1137d 100644 --- a/chromadb/utils/embedding_functions.py +++ b/chromadb/utils/embedding_functions.py @@ -1,5 +1,6 @@ import hashlib import logging +from enum import Enum from functools import cached_property from tenacity import stop_after_attempt, wait_random, retry, retry_if_exception @@ -898,16 +899,20 @@ def __call__(self, input: Documents) -> Embeddings: ) -class VoyageAIEmbeddingFunction(EmbeddingFunction): +class VoyageAIEmbeddingFunction(EmbeddingFunction[Documents]): """Embedding function for Voyageai.com. API docs - https://docs.voyageai.com/reference/embeddings-api""" + class InputType(str, Enum): + DOCUMENT = "document" + QUERY = "query" + def __init__( self, - api_key: str, + api_key: Optional[str] = None, model_name: str = "voyage-2", max_batch_size: int = 128, truncation: Optional[bool] = True, - input_type: Optional[str] = None, + input_type: Optional[InputType] = None, ): """ Initialize the VoyageAIEmbeddingFunction. @@ -919,7 +924,7 @@ def __init__( input_type (str, optional): The type of input text. Can be `None`, `query`, `document`. Defaults to `None`. """ - if not api_key: + if not api_key and "VOYAGE_API_KEY" not in os.environ: raise ValueError("Please provide a VoyageAI API key.") try: @@ -929,11 +934,10 @@ def __init__( raise ValueError( f"The maximum batch size supported is {voyageai.VOYAGE_EMBED_BATCH_SIZE}." ) - voyageai.api_key = api_key # Voyage API Key self._batch_size = max_batch_size self._model = model_name self._truncation = truncation - self._client = voyageai.Client() + self._client = voyageai.Client(api_key=api_key) self._input_type = input_type except ImportError: raise ValueError( @@ -960,7 +964,7 @@ def __call__(self, input: Documents) -> Embeddings: truncation=self._truncation, input_type=self._input_type, ) - return results.embeddings + return cast(Embeddings, results.embeddings) def create_langchain_embedding(langchain_embdding_fn: Any): # type: ignore From 104ef71d4436a0cc7d218543d86eaca7d445b21f Mon Sep 17 00:00:00 2001 From: Trayan Azarov Date: Fri, 21 Jun 2024 16:51:24 +0200 Subject: [PATCH 09/12] feat: Rebase + minor test improvement --- chromadb/test/ef/test_voyageai.py | 66 ++++++++++++---- .../voyage_ai_embedding_function.py | 77 +++++++++++++++++++ 2 files changed, 130 insertions(+), 13 deletions(-) create mode 100644 chromadb/utils/embedding_functions/voyage_ai_embedding_function.py diff --git a/chromadb/test/ef/test_voyageai.py b/chromadb/test/ef/test_voyageai.py index ced093192d9..5baaf70fe26 100644 --- a/chromadb/test/ef/test_voyageai.py +++ b/chromadb/test/ef/test_voyageai.py @@ -2,7 +2,11 @@ import pytest -from chromadb.utils.embedding_functions import VoyageAIEmbeddingFunction +from chromadb.utils.embedding_functions.voyage_ai_embedding_function import ( + VoyageAIEmbeddingFunction, +) + +voyageai = pytest.importorskip("voyageai", reason="voyageai not installed") @pytest.fixture(scope="function") @@ -18,7 +22,10 @@ def remove_api_key(): os.environ["VOYAGE_API_KEY"] = existing_api_key -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage() -> None: ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGE_API_KEY", "")) embeddings = ef(["test doc"]) @@ -27,7 +34,10 @@ def test_voyage() -> None: assert len(embeddings[0]) > 0 -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_input_type_query() -> None: ef = VoyageAIEmbeddingFunction( api_key=os.environ.get("VOYAGE_API_KEY", ""), @@ -39,7 +49,10 @@ def test_voyage_input_type_query() -> None: assert len(embeddings[0]) > 0 -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_input_type_document() -> None: ef = VoyageAIEmbeddingFunction( api_key=os.environ.get("VOYAGE_API_KEY", ""), @@ -51,7 +64,10 @@ def test_voyage_input_type_document() -> None: assert len(embeddings[0]) > 0 -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_model() -> None: ef = VoyageAIEmbeddingFunction( api_key=os.environ.get("VOYAGE_API_KEY", ""), model_name="voyage-01" @@ -62,7 +78,10 @@ def test_voyage_model() -> None: assert len(embeddings[0]) > 0 -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_truncation_default() -> None: ef = VoyageAIEmbeddingFunction(api_key=os.environ.get("VOYAGE_API_KEY", "")) embeddings = ef(["this is a test-message" * 10000]) @@ -71,7 +90,10 @@ def test_voyage_truncation_default() -> None: assert len(embeddings[0]) > 0 -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_truncation_enabled() -> None: ef = VoyageAIEmbeddingFunction( api_key=os.environ.get("VOYAGE_API_KEY", ""), truncation=True @@ -82,7 +104,10 @@ def test_voyage_truncation_enabled() -> None: assert len(embeddings[0]) > 0 -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_truncation_disabled() -> None: ef = VoyageAIEmbeddingFunction( api_key=os.environ.get("VOYAGE_API_KEY", ""), truncation=False @@ -91,30 +116,45 @@ def test_voyage_truncation_disabled() -> None: ef(["this is a test-message" * 10000]) -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_env_api_key() -> None: VoyageAIEmbeddingFunction() -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_no_api_key(remove_api_key) -> None: with pytest.raises(ValueError, match="Please provide a VoyageAI API key"): VoyageAIEmbeddingFunction(api_key=None) # type: ignore -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_no_api_key_in_env(remove_api_key) -> None: with pytest.raises(ValueError, match="Please provide a VoyageAI API key"): VoyageAIEmbeddingFunction(api_key=None) # type: ignore -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_max_batch_size_exceeded_in_init() -> None: with pytest.raises(ValueError, match="The maximum batch size supported is"): VoyageAIEmbeddingFunction(api_key="dummy", max_batch_size=99999999) -@pytest.mark.skipif("VOYAGE_API_KEY" not in os.environ, reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.") +@pytest.mark.skipif( + "VOYAGE_API_KEY" not in os.environ, + reason="VOYAGE_API_KEY not set, not going to test VoyageAI EF.", +) def test_voyage_max_batch_size_exceeded_in_call() -> None: ef = VoyageAIEmbeddingFunction(api_key="dummy", max_batch_size=1) with pytest.raises(ValueError, match="The maximum batch size supported is"): diff --git a/chromadb/utils/embedding_functions/voyage_ai_embedding_function.py b/chromadb/utils/embedding_functions/voyage_ai_embedding_function.py new file mode 100644 index 00000000000..cbe1534bc70 --- /dev/null +++ b/chromadb/utils/embedding_functions/voyage_ai_embedding_function.py @@ -0,0 +1,77 @@ +import os +from enum import Enum +from typing import Optional, cast + +from chromadb.api.types import ( + Documents, + EmbeddingFunction, + Embeddings, +) + + +class VoyageAIEmbeddingFunction(EmbeddingFunction[Documents]): + """Embedding function for Voyageai.com. API docs - https://docs.voyageai.com/reference/embeddings-api""" + + class InputType(str, Enum): + DOCUMENT = "document" + QUERY = "query" + + def __init__( + self, + api_key: Optional[str] = None, + model_name: str = "voyage-2", + max_batch_size: int = 128, + truncation: Optional[bool] = True, + input_type: Optional[InputType] = None, + ): + """ + Initialize the VoyageAIEmbeddingFunction. + Args: + api_key (str): Your API key for the HuggingFace API. + model_name (str, optional): The name of the model to use for text embeddings. Defaults to "voyage-01". + batch_size (int, optional): The number of documents to send at a time. Defaults to 128 (The max supported 7th Apr 2024). see voyageai.VOYAGE_EMBED_BATCH_SIZE for actual max. + truncation (bool, optional): Whether to truncate the input (`True`) or raise an error if the input is too long (`False`). Defaults to `False`. + input_type (str, optional): The type of input text. Can be `None`, `query`, `document`. Defaults to `None`. + """ + + if not api_key and "VOYAGE_API_KEY" not in os.environ: + raise ValueError("Please provide a VoyageAI API key.") + + try: + import voyageai + + if max_batch_size > voyageai.VOYAGE_EMBED_BATCH_SIZE: + raise ValueError( + f"The maximum batch size supported is {voyageai.VOYAGE_EMBED_BATCH_SIZE}." + ) + self._batch_size = max_batch_size + self._model = model_name + self._truncation = truncation + self._client = voyageai.Client(api_key=api_key) + self._input_type = input_type + except ImportError: + raise ValueError( + "The VoyageAI python package is not installed. Please install it with `pip install voyageai`" + ) + + def __call__(self, input: Documents) -> Embeddings: + """ + Get the embeddings for a list of texts. + Args: + input (Documents): A list of texts to get embeddings for. + Returns: + Embeddings: The embeddings for the texts. + Example: + >>> voyage_ef = VoyageAIEmbeddingFunction(api_key="your_api_key") + >>> input = ["Hello, world!", "How are you?"] + >>> embeddings = voyage_ef(input) + """ + if len(input) > self._batch_size: + raise ValueError(f"The maximum batch size supported is {self._batch_size}.") + results = self._client.embed( + texts=input, + model=self._model, + truncation=self._truncation, + input_type=self._input_type, + ) + return cast(Embeddings, results.embeddings) From 9b3a1778a63c2c0df216bee3f5e0163db9f09f27 Mon Sep 17 00:00:00 2001 From: Trayan Azarov Date: Fri, 21 Jun 2024 17:05:49 +0200 Subject: [PATCH 10/12] docs: Updating docs with PR from docs repo Refs: chroma-core/docs#226 --- .../pages/guides/embeddings.md | 17 ++++---- .../pages/integrations/_sidenav.js | 1 + .../pages/integrations/voyageai.md | 42 +++++++++++++++++++ 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 docs/docs.trychroma.com/pages/integrations/voyageai.md diff --git a/docs/docs.trychroma.com/pages/guides/embeddings.md b/docs/docs.trychroma.com/pages/guides/embeddings.md index d523c7d3089..2f45ca20e4f 100644 --- a/docs/docs.trychroma.com/pages/guides/embeddings.md +++ b/docs/docs.trychroma.com/pages/guides/embeddings.md @@ -9,15 +9,16 @@ Chroma provides lightweight wrappers around popular embedding providers, making {% special_table %} {% /special_table %} -| | Python | JS | -|--------------|-----------|---------------| -| [OpenAI](/integrations/openai) | ✅ | ✅ | -| [Google Generative AI](/integrations/google-gemini) | ✅ | ✅ | -| [Cohere](/integrations/cohere) | ✅ | ✅ | -| [Hugging Face](/integrations/hugging-face) | ✅ | ➖ | -| [Instructor](/integrations/instructor) | ✅ | ➖ | +| | Python | JS | +|--------------------------------------------------------------------|-----------|---------------| +| [OpenAI](/integrations/openai) | ✅ | ✅ | +| [Google Generative AI](/integrations/google-gemini) | ✅ | ✅ | +| [Cohere](/integrations/cohere) | ✅ | ✅ | +| [Hugging Face](/integrations/hugging-face) | ✅ | ➖ | +| [Instructor](/integrations/instructor) | ✅ | ➖ | | [Hugging Face Embedding Server](/integrations/hugging-face-server) | ✅ | ✅ | -| [Jina AI](/integrations/jinaai) | ✅ | ✅ | +| [Jina AI](/integrations/jinaai) | ✅ | ✅ | +| [Voyage AI](/integrations/voyageai) | ✅ | ✅ | We welcome pull requests to add new Embedding Functions to the community. diff --git a/docs/docs.trychroma.com/pages/integrations/_sidenav.js b/docs/docs.trychroma.com/pages/integrations/_sidenav.js index 5feeda2d186..eddc168a704 100644 --- a/docs/docs.trychroma.com/pages/integrations/_sidenav.js +++ b/docs/docs.trychroma.com/pages/integrations/_sidenav.js @@ -11,6 +11,7 @@ export const items = [ { href: '/integrations/jinaai', children: 'JinaAI' }, { href: '/integrations/roboflow', children: 'Roboflow' }, { href: '/integrations/ollama', children: 'Ollama Embeddings' }, + { href: '/integrations/voyageai', children: 'Voyage AI Embeddings' }, ] }, { diff --git a/docs/docs.trychroma.com/pages/integrations/voyageai.md b/docs/docs.trychroma.com/pages/integrations/voyageai.md new file mode 100644 index 00000000000..13bcf7f62d5 --- /dev/null +++ b/docs/docs.trychroma.com/pages/integrations/voyageai.md @@ -0,0 +1,42 @@ +--- +title: Voyage AI Embeddings +--- + +Chroma also provides a convenient wrapper around VoyageAI's embedding API. This embedding function runs remotely on VoyageAI’s servers, and requires an API key. You can get an API key by signing up for an account at [VoyageAI](https://dash.voyageai.com/api-keys). + +{% tabs group="code-lang" %} +{% tab label="Python" %} + +This embedding function relies on the `voyageai` python package, which you can install with `pip install voyageai`. + +```python +from chromadb.utils.embedding_functions.voyage_ai_embedding_function import VoyageAIEmbeddingFunction +voyageai_ef = VoyageAIEmbeddingFunction(api_key="YOUR_API_KEY", model_name="voyage-law-2", input_type=VoyageAIEmbeddingFunction.InputType.DOCUMENT) +result = voyageai_ef(input=["document1","document2"]) +``` + +{% /tab %} +{% tab label="Javascript" %} + +```javascript +const {VoyageAIEmbeddingFunction, InputType} = require('chromadb'); +// const {VoyageAIEmbeddingFunction, InputType} from "chromadb"; // ESM import +const embedder = new VoyageAIEmbeddingFunction("apiKey", "voyage-law-2", InputType.DOCUMENT) + +// use directly +const embeddings = embedder.generate(["document1","document2"]) + +// pass documents to query for .add and .query +const collection = await client.createCollection({name: "name", embeddingFunction: embedder}) +const collectionGet = await client.getCollection({name:"name", embeddingFunction: embedder}) +``` + +{% /codetab %} +{% /codetabs %} + +{% /tab %} + +{% /tabs %} + +You should pass in the `model_name` argument, which lets you choose which VoyageAI embeddings model to use. You can see the available models [here](https://docs.voyageai.com/docs/embeddings). + From 28ff847930f7580767011bb42e539f8d14ecd2c4 Mon Sep 17 00:00:00 2001 From: Trayan Azarov Date: Fri, 21 Jun 2024 17:07:41 +0200 Subject: [PATCH 11/12] docs: Fixed docs linting --- docs/docs.trychroma.com/pages/integrations/voyageai.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/docs.trychroma.com/pages/integrations/voyageai.md b/docs/docs.trychroma.com/pages/integrations/voyageai.md index 13bcf7f62d5..671161b0c10 100644 --- a/docs/docs.trychroma.com/pages/integrations/voyageai.md +++ b/docs/docs.trychroma.com/pages/integrations/voyageai.md @@ -23,7 +23,7 @@ const {VoyageAIEmbeddingFunction, InputType} = require('chromadb'); // const {VoyageAIEmbeddingFunction, InputType} from "chromadb"; // ESM import const embedder = new VoyageAIEmbeddingFunction("apiKey", "voyage-law-2", InputType.DOCUMENT) -// use directly +// use directly const embeddings = embedder.generate(["document1","document2"]) // pass documents to query for .add and .query @@ -39,4 +39,3 @@ const collectionGet = await client.getCollection({name:"name", embeddingFunction {% /tabs %} You should pass in the `model_name` argument, which lets you choose which VoyageAI embeddings model to use. You can see the available models [here](https://docs.voyageai.com/docs/embeddings). - From 7c713bd47f56448e81e9b5beddf576f8272c19c8 Mon Sep 17 00:00:00 2001 From: Trayan Azarov Date: Fri, 21 Jun 2024 17:23:47 +0200 Subject: [PATCH 12/12] fix: Fixing test_ef.py to also account for VoyageAIEmbeddingFunction --- chromadb/test/ef/test_ef.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chromadb/test/ef/test_ef.py b/chromadb/test/ef/test_ef.py index c93502e3fc8..c3541c4801f 100644 --- a/chromadb/test/ef/test_ef.py +++ b/chromadb/test/ef/test_ef.py @@ -30,6 +30,7 @@ def test_get_builtins_holds() -> None: "SentenceTransformerEmbeddingFunction", "Text2VecEmbeddingFunction", "ChromaLangchainEmbeddingFunction", + "VoyageAIEmbeddingFunction", } assert expected_builtins == embedding_functions.get_builtins()