feat: binary vector implementation.

README.md

@@ -34,6 +34,10 @@ The following table represents the association between Erlang types and BSON typ
 | `[ tuple() \| _ ] ` | Object (3) |
 | `list()` | Array (4) |
 | `{data, binary, binary()}` | Binary data (5) |
+| `{vector, int8, [integer()]}` | Binary Vector INT8 (5, subtype 9) |
+| `{vector, float32, [float()]}` | Binary Vector FLOAT32 (5, subtype 9) |
+| `{vector, packed_bit, binary()}` | Binary Vector PACKED_BIT (5, subtype 9) |
+| `{vector, packed_bit, binary(), 0..7}` | Binary Vector PACKED_BIT with padding (5, subtype 9) |
 | `undefined` | Undefined (6) |
 | `{object_id, <<_:96>>}` | ObjectId (7) |
 | `false \| true` | Boolean (8) |
@@ -53,6 +57,29 @@ The following table represents the association between Erlang types and BSON typ
 
 On deserialization, we prevent the dynamic generation of atoms by converting BSON Symbol, Max Key and Min Key values to Erlang binaries.
 
+### Binary Vectors
+
+`nbson` supports [BSON Binary Subtype 9](https://github.com/mongodb/specifications/blob/master/source/bson-binary-vector/bson-binary-vector.md) for efficient storage of dense numeric vectors, useful for AI/ML applications and semantic search.
+
+| Vector Type | Erlang Representation | Value Range |
+| ----------- | --------------------- | ----------- |
+| INT8 | `{vector, int8, [integer()]}` | -128 to 127 |
+| FLOAT32 | `{vector, float32, [float()]}` | IEEE 754 single-precision (supports `infinity`, `neg_infinity`) |
+| PACKED_BIT | `{vector, packed_bit, binary()}` | Raw bytes (0-255) representing packed bits |
+| PACKED_BIT | `{vector, packed_bit, binary(), Padding}` | With padding (0-7 bits to ignore in final byte) |
+
+```erlang
+%% INT8 vector
+{ok, Bin} = nbson:encode(#{<<"embedding">> => {vector, int8, [127, -128, 0, 42]}}).
+{ok, #{<<"embedding">> := {vector, int8, [127, -128, 0, 42]}}} = nbson:decode(Bin).
+
+%% FLOAT32 vector with special values
+{ok, Bin} = nbson:encode(#{<<"v">> => {vector, float32, [1.5, infinity, neg_infinity]}}).
+
+%% PACKED_BIT vector with padding (13 bits = 2 bytes with 3 bits padding)
+{ok, Bin} = nbson:encode(#{<<"bits">> => {vector, packed_bit, <<255, 248>>, 3}}).
+```
+
 ## Benchmarking
 The BSON decoder implementation in `nbson_decoder.erl` uses [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style). In this particular case, CPS leads to the use of the [sub binary delayed optimization](https://www.erlang.org/doc/efficiency_guide/binaryhandling.html#match-context) and improved efficiency in the deserialization process.
 

src/nbson.erl

@@ -23,6 +23,14 @@
 -type document_path() :: [key()].
 -type key() :: binary().
 -type regex_arg() :: unicode:latin1_chardata() | unicode:chardata() | unicode:external_chardata().
+-type int8() :: -128..127.
+-type vector_padding() :: 0..7.
+-type float32_value() :: float() | integer() | infinity | neg_infinity.
+-type vector() ::
+    {vector, int8, [integer()]}
+    | {vector, float32, [float32_value()]}
+    | {vector, packed_bit, binary()}
+    | {vector, packed_bit, binary(), vector_padding()}.
 -type value() ::
     float()
     | integer()
@@ -44,18 +52,28 @@
     | {data, encrypted, binary()}
     | {data, compressed, binary()}
     | {data, user, binary()}
+    | vector()
     | {object_id, binary()}
     | erlang:timestamp()
     | {regex, regex_arg(), regex_arg()}
     | {pointer, binary(), binary()}
     | {javascript, map(), binary()}
     | {javascript, document(), binary()}
     | {timestamp, non_neg_integer(), non_neg_integer()}.
--type decode_error_reason() :: invalid_subtype | invalid_bson.
+-type vector_error_reason() ::
+    {invalid_vector_dtype, non_neg_integer()}
+    | {invalid_vector_padding, integer()}
+    | {invalid_vector_int8_padding, non_neg_integer()}
+    | {invalid_vector_float32_padding, non_neg_integer()}
+    | {invalid_vector_float32_length, non_neg_integer()}
+    | {invalid_vector_packed_bit_empty_with_padding, non_neg_integer()}
+    | {invalid_vector_int8_value, integer()}.
+-type decode_error_reason() :: invalid_subtype | invalid_bson | vector_error_reason().
 -type encode_error_reason() ::
     {invalid_proplist_document, term()}
     | {not_unicode_regex, {term(), term()}}
-    | {integer_too_large, integer()}.
+    | {integer_too_large, integer()}
+    | vector_error_reason().
 
 %%% EXPORT TYPES
 -export_type([
@@ -65,8 +83,13 @@
     document_path/0,
     key/0,
     value/0,
+    vector/0,
+    int8/0,
+    float32_value/0,
+    vector_padding/0,
     decode_error_reason/0,
-    encode_error_reason/0
+    encode_error_reason/0,
+    vector_error_reason/0
 ]).
 
 %%%-----------------------------------------------------------------------------

src/nbson_decoder.erl

@@ -246,6 +246,14 @@ string(Base, Len, Next) ->
     SubType :: non_neg_integer(),
     Next :: [term()],
     Result :: nbson:document() | {error, nbson:decode_error_reason()}.
+binary(<<Base/binary>>, Size, 9, Next) ->
+    <<Part:Size/binary, Bin/binary>> = Base,
+    case decode_vector(Part) of
+        {error, _} = Error ->
+            Error;
+        Vector ->
+            next(Bin, Vector, Next)
+    end;
 binary(<<Base/binary>>, Size, SubType, Next) ->
     <<Part:Size/binary, Bin/binary>> = Base,
     case subtype_decode(SubType) of
@@ -279,3 +287,61 @@ subtype_decode(128) ->
     user;
 subtype_decode(_) ->
     {error, invalid_subtype}.
+
+%%%-----------------------------------------------------------------------------
+%%% VECTOR DECODING FUNCTIONS
+%%%-----------------------------------------------------------------------------
+-define(VECTOR_DTYPE_INT8, 16#03).
+-define(VECTOR_DTYPE_FLOAT32, 16#27).
+-define(VECTOR_DTYPE_PACKED_BIT, 16#10).
+
+-spec decode_vector(Data) -> Result when
+    Data :: binary(),
+    Result :: nbson:vector() | {error, nbson:vector_error_reason()}.
+decode_vector(<<DType:8, Padding:8, VectorData/binary>>) ->
+    case DType of
+        ?VECTOR_DTYPE_INT8 when Padding =:= 0 ->
+            {vector, int8, decode_int8_values(VectorData, [])};
+        ?VECTOR_DTYPE_INT8 ->
+            {error, {invalid_vector_int8_padding, Padding}};
+        ?VECTOR_DTYPE_FLOAT32 when Padding =:= 0, byte_size(VectorData) rem 4 =:= 0 ->
+            {vector, float32, decode_float32_values(VectorData, [])};
+        ?VECTOR_DTYPE_FLOAT32 when Padding =:= 0 ->
+            {error, {invalid_vector_float32_length, byte_size(VectorData)}};
+        ?VECTOR_DTYPE_FLOAT32 ->
+            {error, {invalid_vector_float32_padding, Padding}};
+        ?VECTOR_DTYPE_PACKED_BIT when Padding > 7 ->
+            {error, {invalid_vector_padding, Padding}};
+        ?VECTOR_DTYPE_PACKED_BIT when Padding > 0, byte_size(VectorData) =:= 0 ->
+            {error, {invalid_vector_packed_bit_empty_with_padding, Padding}};
+        ?VECTOR_DTYPE_PACKED_BIT when Padding =:= 0 ->
+            {vector, packed_bit, VectorData};
+        ?VECTOR_DTYPE_PACKED_BIT ->
+            {vector, packed_bit, VectorData, Padding};
+        _ ->
+            {error, {invalid_vector_dtype, DType}}
+    end.
+
+-spec decode_int8_values(Data, Acc) -> Result when
+    Data :: binary(),
+    Acc :: [integer()],
+    Result :: [integer()].
+decode_int8_values(<<>>, Acc) ->
+    lists:reverse(Acc);
+decode_int8_values(<<V:8/signed, Rest/binary>>, Acc) ->
+    decode_int8_values(Rest, [V | Acc]).
+
+-spec decode_float32_values(Data, Acc) -> Result when
+    Data :: binary(),
+    Acc :: [nbson:float32_value()],
+    Result :: [nbson:float32_value()].
+decode_float32_values(<<>>, Acc) ->
+    lists:reverse(Acc);
+decode_float32_values(<<0, 0, 128, 127, Rest/binary>>, Acc) ->
+    %% Positive infinity (IEEE 754)
+    decode_float32_values(Rest, [infinity | Acc]);
+decode_float32_values(<<0, 0, 128, 255, Rest/binary>>, Acc) ->
+    %% Negative infinity (IEEE 754)
+    decode_float32_values(Rest, [neg_infinity | Acc]);
+decode_float32_values(<<V:32/little-float, Rest/binary>>, Acc) ->
+    decode_float32_values(Rest, [V | Acc]).

src/nbson_encoder.erl

@@ -158,6 +158,16 @@ encode_value({data, compressed, Data}) when is_binary(Data) ->
     {?BIN_TYPE, <<?INT32(byte_size(Data)), ?INT8(7), Data/binary>>};
 encode_value({data, user, Data}) when is_binary(Data) ->
     {?BIN_TYPE, <<?INT32(byte_size(Data)), ?INT8(128), Data/binary>>};
+encode_value({vector, int8, Values}) when is_list(Values) ->
+    encode_vector_int8(Values);
+encode_value({vector, float32, Values}) when is_list(Values) ->
+    encode_vector_float32(Values);
+encode_value({vector, packed_bit, Data}) when is_binary(Data) ->
+    encode_vector_packed_bit(Data, 0);
+encode_value({vector, packed_bit, Data, Padding}) when
+    is_binary(Data), Padding >= 0, Padding =< 7
+->
+    encode_vector_packed_bit(Data, Padding);
 encode_value(undefined) ->
     {?UNDEF_TYPE, <<>>};
 encode_value({object_id, <<_:96>> = Id}) ->
@@ -254,3 +264,71 @@ map_fold_encode(Label, Value, Acc) ->
         {Type, Payload} ->
             <<Acc/binary, ?INT8(Type), ?CSTRING(encode_label(Label)), Payload/binary>>
     end.
+
+%%%-----------------------------------------------------------------------------
+%%% VECTOR ENCODING FUNCTIONS
+%%%-----------------------------------------------------------------------------
+-define(VECTOR_SUBTYPE, 9).
+-define(VECTOR_DTYPE_INT8, 16#03).
+-define(VECTOR_DTYPE_FLOAT32, 16#27).
+-define(VECTOR_DTYPE_PACKED_BIT, 16#10).
+
+-spec encode_vector_int8(Values) -> Result when
+    Values :: [integer()],
+    Result :: {?BIN_TYPE, binary()} | {error, nbson:encode_error_reason()}.
+encode_vector_int8(Values) ->
+    case encode_int8_values(Values) of
+        {error, _Reason} = Error ->
+            Error;
+        Data ->
+            VectorData = <<?INT8(?VECTOR_DTYPE_INT8), ?INT8(0), Data/binary>>,
+            {?BIN_TYPE, <<
+                ?INT32(byte_size(VectorData)), ?INT8(?VECTOR_SUBTYPE), VectorData/binary
+            >>}
+    end.
+
+-spec encode_int8_values(Values) -> Result when
+    Values :: [integer()],
+    Result :: binary() | {error, nbson:encode_error_reason()}.
+encode_int8_values(Values) ->
+    Data = <<<<V:8/signed>> || V <- Values, V >= -128, V =< 127>>,
+    case byte_size(Data) == length(Values) of
+        true ->
+            Data;
+        false ->
+            {value, InvalidValue} = lists:search(fun(V) -> V < -128 orelse V > 127 end, Values),
+            {error, {invalid_vector_int8_value, InvalidValue}}
+    end.
+
+-spec encode_vector_float32(Values) -> Result when
+    Values :: [nbson:float32_value()],
+    Result :: {?BIN_TYPE, binary()}.
+encode_vector_float32(Values) ->
+    Data = encode_float32_values(Values, <<>>),
+    VectorData = <<?INT8(?VECTOR_DTYPE_FLOAT32), ?INT8(0), Data/binary>>,
+    {?BIN_TYPE, <<?INT32(byte_size(VectorData)), ?INT8(?VECTOR_SUBTYPE), VectorData/binary>>}.
+
+-spec encode_float32_values(Values, Acc) -> Result when
+    Values :: [nbson:float32_value()],
+    Acc :: binary(),
+    Result :: binary().
+encode_float32_values([], Acc) ->
+    Acc;
+encode_float32_values([infinity | Rest], Acc) ->
+    %% Positive infinity (IEEE 754 single precision)
+    encode_float32_values(Rest, <<Acc/binary, 0, 0, 128, 127>>);
+encode_float32_values([neg_infinity | Rest], Acc) ->
+    %% Negative infinity (IEEE 754 single precision)
+    encode_float32_values(Rest, <<Acc/binary, 0, 0, 128, 255>>);
+encode_float32_values([V | Rest], Acc) when is_number(V) ->
+    encode_float32_values(Rest, <<Acc/binary, V:32/little-float>>).
+
+-spec encode_vector_packed_bit(Data, Padding) -> Result when
+    Data :: binary(),
+    Padding :: nbson:vector_padding(),
+    Result :: {?BIN_TYPE, binary()} | {error, nbson:encode_error_reason()}.
+encode_vector_packed_bit(<<>>, Padding) when Padding > 0 ->
+    {error, {invalid_vector_packed_bit_empty_with_padding, Padding}};
+encode_vector_packed_bit(Data, Padding) ->
+    VectorData = <<?INT8(?VECTOR_DTYPE_PACKED_BIT), ?INT8(Padding), Data/binary>>,
+    {?BIN_TYPE, <<?INT32(byte_size(VectorData)), ?INT8(?VECTOR_SUBTYPE), VectorData/binary>>}.