diff --git a/README.md b/README.md index fb7a4bd..cbd37b3 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ The following table represents the association between Erlang types and BSON typ | `[ tuple() \| _ ] ` | Object (3) | | `list()` | Array (4) | | `{data, binary, binary()}` | Binary data (5) | +| `{vector, int8, [integer()]}` | Binary Vector INT8 (5, subtype 9) | +| `{vector, float32, [float()]}` | Binary Vector FLOAT32 (5, subtype 9) | +| `{vector, packed_bit, binary()}` | Binary Vector PACKED_BIT (5, subtype 9) | +| `{vector, packed_bit, binary(), 0..7}` | Binary Vector PACKED_BIT with padding (5, subtype 9) | | `undefined` | Undefined (6) | | `{object_id, <<_:96>>}` | ObjectId (7) | | `false \| true` | Boolean (8) | @@ -53,6 +57,29 @@ The following table represents the association between Erlang types and BSON typ On deserialization, we prevent the dynamic generation of atoms by converting BSON Symbol, Max Key and Min Key values to Erlang binaries. +### Binary Vectors + +`nbson` supports [BSON Binary Subtype 9](https://github.com/mongodb/specifications/blob/master/source/bson-binary-vector/bson-binary-vector.md) for efficient storage of dense numeric vectors, useful for AI/ML applications and semantic search. + +| Vector Type | Erlang Representation | Value Range | +| ----------- | --------------------- | ----------- | +| INT8 | `{vector, int8, [integer()]}` | -128 to 127 | +| FLOAT32 | `{vector, float32, [float()]}` | IEEE 754 single-precision (supports `infinity`, `neg_infinity`) | +| PACKED_BIT | `{vector, packed_bit, binary()}` | Raw bytes (0-255) representing packed bits | +| PACKED_BIT | `{vector, packed_bit, binary(), Padding}` | With padding (0-7 bits to ignore in final byte) | + +```erlang +%% INT8 vector +{ok, Bin} = nbson:encode(#{<<"embedding">> => {vector, int8, [127, -128, 0, 42]}}). +{ok, #{<<"embedding">> := {vector, int8, [127, -128, 0, 42]}}} = nbson:decode(Bin). + +%% FLOAT32 vector with special values +{ok, Bin} = nbson:encode(#{<<"v">> => {vector, float32, [1.5, infinity, neg_infinity]}}). + +%% PACKED_BIT vector with padding (13 bits = 2 bytes with 3 bits padding) +{ok, Bin} = nbson:encode(#{<<"bits">> => {vector, packed_bit, <<255, 248>>, 3}}). +``` + ## Benchmarking The BSON decoder implementation in `nbson_decoder.erl` uses [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style). In this particular case, CPS leads to the use of the [sub binary delayed optimization](https://www.erlang.org/doc/efficiency_guide/binaryhandling.html#match-context) and improved efficiency in the deserialization process. diff --git a/src/nbson.erl b/src/nbson.erl index cd30322..1c63c42 100644 --- a/src/nbson.erl +++ b/src/nbson.erl @@ -23,6 +23,14 @@ -type document_path() :: [key()]. -type key() :: binary(). -type regex_arg() :: unicode:latin1_chardata() | unicode:chardata() | unicode:external_chardata(). +-type int8() :: -128..127. +-type vector_padding() :: 0..7. +-type float32_value() :: float() | integer() | infinity | neg_infinity. +-type vector() :: + {vector, int8, [integer()]} + | {vector, float32, [float32_value()]} + | {vector, packed_bit, binary()} + | {vector, packed_bit, binary(), vector_padding()}. -type value() :: float() | integer() @@ -44,6 +52,7 @@ | {data, encrypted, binary()} | {data, compressed, binary()} | {data, user, binary()} + | vector() | {object_id, binary()} | erlang:timestamp() | {regex, regex_arg(), regex_arg()} @@ -51,11 +60,20 @@ | {javascript, map(), binary()} | {javascript, document(), binary()} | {timestamp, non_neg_integer(), non_neg_integer()}. --type decode_error_reason() :: invalid_subtype | invalid_bson. +-type vector_error_reason() :: + {invalid_vector_dtype, non_neg_integer()} + | {invalid_vector_padding, integer()} + | {invalid_vector_int8_padding, non_neg_integer()} + | {invalid_vector_float32_padding, non_neg_integer()} + | {invalid_vector_float32_length, non_neg_integer()} + | {invalid_vector_packed_bit_empty_with_padding, non_neg_integer()} + | {invalid_vector_int8_value, integer()}. +-type decode_error_reason() :: invalid_subtype | invalid_bson | vector_error_reason(). -type encode_error_reason() :: {invalid_proplist_document, term()} | {not_unicode_regex, {term(), term()}} - | {integer_too_large, integer()}. + | {integer_too_large, integer()} + | vector_error_reason(). %%% EXPORT TYPES -export_type([ @@ -65,8 +83,13 @@ document_path/0, key/0, value/0, + vector/0, + int8/0, + float32_value/0, + vector_padding/0, decode_error_reason/0, - encode_error_reason/0 + encode_error_reason/0, + vector_error_reason/0 ]). %%%----------------------------------------------------------------------------- diff --git a/src/nbson_decoder.erl b/src/nbson_decoder.erl index 732f39b..fc02c7a 100644 --- a/src/nbson_decoder.erl +++ b/src/nbson_decoder.erl @@ -246,6 +246,14 @@ string(Base, Len, Next) -> SubType :: non_neg_integer(), Next :: [term()], Result :: nbson:document() | {error, nbson:decode_error_reason()}. +binary(<>, Size, 9, Next) -> + <> = Base, + case decode_vector(Part) of + {error, _} = Error -> + Error; + Vector -> + next(Bin, Vector, Next) + end; binary(<>, Size, SubType, Next) -> <> = Base, case subtype_decode(SubType) of @@ -279,3 +287,61 @@ subtype_decode(128) -> user; subtype_decode(_) -> {error, invalid_subtype}. + +%%%----------------------------------------------------------------------------- +%%% VECTOR DECODING FUNCTIONS +%%%----------------------------------------------------------------------------- +-define(VECTOR_DTYPE_INT8, 16#03). +-define(VECTOR_DTYPE_FLOAT32, 16#27). +-define(VECTOR_DTYPE_PACKED_BIT, 16#10). + +-spec decode_vector(Data) -> Result when + Data :: binary(), + Result :: nbson:vector() | {error, nbson:vector_error_reason()}. +decode_vector(<>) -> + case DType of + ?VECTOR_DTYPE_INT8 when Padding =:= 0 -> + {vector, int8, decode_int8_values(VectorData, [])}; + ?VECTOR_DTYPE_INT8 -> + {error, {invalid_vector_int8_padding, Padding}}; + ?VECTOR_DTYPE_FLOAT32 when Padding =:= 0, byte_size(VectorData) rem 4 =:= 0 -> + {vector, float32, decode_float32_values(VectorData, [])}; + ?VECTOR_DTYPE_FLOAT32 when Padding =:= 0 -> + {error, {invalid_vector_float32_length, byte_size(VectorData)}}; + ?VECTOR_DTYPE_FLOAT32 -> + {error, {invalid_vector_float32_padding, Padding}}; + ?VECTOR_DTYPE_PACKED_BIT when Padding > 7 -> + {error, {invalid_vector_padding, Padding}}; + ?VECTOR_DTYPE_PACKED_BIT when Padding > 0, byte_size(VectorData) =:= 0 -> + {error, {invalid_vector_packed_bit_empty_with_padding, Padding}}; + ?VECTOR_DTYPE_PACKED_BIT when Padding =:= 0 -> + {vector, packed_bit, VectorData}; + ?VECTOR_DTYPE_PACKED_BIT -> + {vector, packed_bit, VectorData, Padding}; + _ -> + {error, {invalid_vector_dtype, DType}} + end. + +-spec decode_int8_values(Data, Acc) -> Result when + Data :: binary(), + Acc :: [integer()], + Result :: [integer()]. +decode_int8_values(<<>>, Acc) -> + lists:reverse(Acc); +decode_int8_values(<>, Acc) -> + decode_int8_values(Rest, [V | Acc]). + +-spec decode_float32_values(Data, Acc) -> Result when + Data :: binary(), + Acc :: [nbson:float32_value()], + Result :: [nbson:float32_value()]. +decode_float32_values(<<>>, Acc) -> + lists:reverse(Acc); +decode_float32_values(<<0, 0, 128, 127, Rest/binary>>, Acc) -> + %% Positive infinity (IEEE 754) + decode_float32_values(Rest, [infinity | Acc]); +decode_float32_values(<<0, 0, 128, 255, Rest/binary>>, Acc) -> + %% Negative infinity (IEEE 754) + decode_float32_values(Rest, [neg_infinity | Acc]); +decode_float32_values(<>, Acc) -> + decode_float32_values(Rest, [V | Acc]). diff --git a/src/nbson_encoder.erl b/src/nbson_encoder.erl index 961aff1..9e7074a 100644 --- a/src/nbson_encoder.erl +++ b/src/nbson_encoder.erl @@ -158,6 +158,16 @@ encode_value({data, compressed, Data}) when is_binary(Data) -> {?BIN_TYPE, <>}; encode_value({data, user, Data}) when is_binary(Data) -> {?BIN_TYPE, <>}; +encode_value({vector, int8, Values}) when is_list(Values) -> + encode_vector_int8(Values); +encode_value({vector, float32, Values}) when is_list(Values) -> + encode_vector_float32(Values); +encode_value({vector, packed_bit, Data}) when is_binary(Data) -> + encode_vector_packed_bit(Data, 0); +encode_value({vector, packed_bit, Data, Padding}) when + is_binary(Data), Padding >= 0, Padding =< 7 +-> + encode_vector_packed_bit(Data, Padding); encode_value(undefined) -> {?UNDEF_TYPE, <<>>}; encode_value({object_id, <<_:96>> = Id}) -> @@ -254,3 +264,71 @@ map_fold_encode(Label, Value, Acc) -> {Type, Payload} -> <> end. + +%%%----------------------------------------------------------------------------- +%%% VECTOR ENCODING FUNCTIONS +%%%----------------------------------------------------------------------------- +-define(VECTOR_SUBTYPE, 9). +-define(VECTOR_DTYPE_INT8, 16#03). +-define(VECTOR_DTYPE_FLOAT32, 16#27). +-define(VECTOR_DTYPE_PACKED_BIT, 16#10). + +-spec encode_vector_int8(Values) -> Result when + Values :: [integer()], + Result :: {?BIN_TYPE, binary()} | {error, nbson:encode_error_reason()}. +encode_vector_int8(Values) -> + case encode_int8_values(Values) of + {error, _Reason} = Error -> + Error; + Data -> + VectorData = <>, + {?BIN_TYPE, << + ?INT32(byte_size(VectorData)), ?INT8(?VECTOR_SUBTYPE), VectorData/binary + >>} + end. + +-spec encode_int8_values(Values) -> Result when + Values :: [integer()], + Result :: binary() | {error, nbson:encode_error_reason()}. +encode_int8_values(Values) -> + Data = <<<> || V <- Values, V >= -128, V =< 127>>, + case byte_size(Data) == length(Values) of + true -> + Data; + false -> + {value, InvalidValue} = lists:search(fun(V) -> V < -128 orelse V > 127 end, Values), + {error, {invalid_vector_int8_value, InvalidValue}} + end. + +-spec encode_vector_float32(Values) -> Result when + Values :: [nbson:float32_value()], + Result :: {?BIN_TYPE, binary()}. +encode_vector_float32(Values) -> + Data = encode_float32_values(Values, <<>>), + VectorData = <>, + {?BIN_TYPE, <>}. + +-spec encode_float32_values(Values, Acc) -> Result when + Values :: [nbson:float32_value()], + Acc :: binary(), + Result :: binary(). +encode_float32_values([], Acc) -> + Acc; +encode_float32_values([infinity | Rest], Acc) -> + %% Positive infinity (IEEE 754 single precision) + encode_float32_values(Rest, <>); +encode_float32_values([neg_infinity | Rest], Acc) -> + %% Negative infinity (IEEE 754 single precision) + encode_float32_values(Rest, <>); +encode_float32_values([V | Rest], Acc) when is_number(V) -> + encode_float32_values(Rest, <>). + +-spec encode_vector_packed_bit(Data, Padding) -> Result when + Data :: binary(), + Padding :: nbson:vector_padding(), + Result :: {?BIN_TYPE, binary()} | {error, nbson:encode_error_reason()}. +encode_vector_packed_bit(<<>>, Padding) when Padding > 0 -> + {error, {invalid_vector_packed_bit_empty_with_padding, Padding}}; +encode_vector_packed_bit(Data, Padding) -> + VectorData = <>, + {?BIN_TYPE, <>}. diff --git a/test/nbson_SUITE.erl b/test/nbson_SUITE.erl index ebddc69..f7d4d89 100644 --- a/test/nbson_SUITE.erl +++ b/test/nbson_SUITE.erl @@ -46,7 +46,18 @@ all() -> timestamp, undef, proplists, - various + various, + vector_int8, + vector_int8_empty, + vector_int8_errors, + vector_float32, + vector_float32_empty, + vector_float32_infinity, + vector_float32_errors, + vector_packed_bit, + vector_packed_bit_with_padding, + vector_packed_bit_empty, + vector_packed_bit_errors ]. %%%----------------------------------------------------------------------------- @@ -362,3 +373,154 @@ various(_Config) -> undefined = nbson:get([<<"arr">>, <<"three">>], #{<<"arr">> => #{<<"two">> => <<"three">>}}), <<"three">> = nbson:get([<<"arr">>, <<"two">>], [{<<"arr">>, [{<<"two">>, <<"three">>}]}]), undefined = nbson:get([<<"arr">>, <<"three">>], [{<<"arr">>, [{<<"two">>, <<"three">>}]}]). + +%%%----------------------------------------------------------------------------- +%%% VECTOR TEST CASES +%%%----------------------------------------------------------------------------- +vector_int8() -> + [{userdata, [{doc, "Tests vector int8 data type BSON encoding/decoding."}]}]. +vector_int8(_Config) -> + %% Test case from spec: Simple Vector INT8 [127, 7] + %% BSON hex: 1600000005766563746F7200040000000903007F0700 + BaseBin = <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 3, 0, 127, 7, 0>>, + BaseMap = #{<<"vector">> => {vector, int8, [127, 7]}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap), + %% Test negative values + NegMap = #{<<"vector">> => {vector, int8, [-128, -1, 0, 1, 127]}}, + {ok, NegBin} = nbson:encode(NegMap), + {ok, NegMap} = nbson:decode(NegBin). + +vector_int8_empty() -> + [{userdata, [{doc, "Tests empty vector int8 data type BSON encoding/decoding."}]}]. +vector_int8_empty(_Config) -> + %% Test case from spec: Empty Vector INT8 [] + %% BSON hex: 1400000005766563746F72000200000009030000 + BaseBin = <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 3, 0, 0>>, + BaseMap = #{<<"vector">> => {vector, int8, []}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_int8_errors() -> + [{userdata, [{doc, "Tests vector int8 error handling."}]}]. +vector_int8_errors(_Config) -> + %% INT8 value overflow (128 is out of range) + {error, {invalid_vector_int8_value, 128}} = + nbson:encode(#{<<"vector">> => {vector, int8, [128]}}), + %% INT8 value underflow (-129 is out of range) + {error, {invalid_vector_int8_value, -129}} = + nbson:encode(#{<<"vector">> => {vector, int8, [-129]}}), + %% INT8 with padding (invalid BSON) + %% BSON hex: 1600000005766563746F7200040000000903037F0700 (padding=3) + InvalidPaddingBin = + <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 3, 3, 127, 7, 0>>, + {error, {invalid_vector_int8_padding, 3}} = nbson:decode(InvalidPaddingBin). + +vector_float32() -> + [{userdata, [{doc, "Tests vector float32 data type BSON encoding/decoding."}]}]. +vector_float32(_Config) -> + %% Test case from spec: Simple Vector FLOAT32 [127.0, 7.0] + %% BSON hex: 1C00000005766563746F72000A0000000927000000FE420000E04000 + BaseBin = + <<28, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 10, 0, 0, 0, 9, 39, 0, 0, 0, 254, 66, 0, + 0, 224, 64, 0>>, + BaseMap = #{<<"vector">> => {vector, float32, [127.0, 7.0]}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap), + %% Test with decimals and negative value: [127.7, -7.7] + %% BSON hex: 1C00000005766563746F72000A0000000927006666FF426666F6C000 + DecBin = + <<28, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 10, 0, 0, 0, 9, 39, 0, 102, 102, 255, 66, + 102, 102, 246, 192, 0>>, + DecMap = #{<<"vector">> => {vector, float32, [127.69999694824219, -7.699999809265137]}}, + {ok, DecMap} = nbson:decode(DecBin). + +vector_float32_empty() -> + [{userdata, [{doc, "Tests empty vector float32 data type BSON encoding/decoding."}]}]. +vector_float32_empty(_Config) -> + %% Test case from spec: Empty Vector FLOAT32 [] + %% BSON hex: 1400000005766563746F72000200000009270000 + BaseBin = <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 39, 0, 0>>, + BaseMap = #{<<"vector">> => {vector, float32, []}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_float32_infinity() -> + [{userdata, [{doc, "Tests vector float32 with infinity values."}]}]. +vector_float32_infinity(_Config) -> + %% Test case from spec: Infinity Vector FLOAT32 [-Infinity, 0.0, Infinity] + %% BSON hex: 2000000005766563746F72000E000000092700000080FF000000000000807F00 + BaseBin = + <<32, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 14, 0, 0, 0, 9, 39, 0, 0, 0, 128, 255, 0, + 0, 0, 0, 0, 0, 128, 127, 0>>, + BaseMap = #{<<"vector">> => {vector, float32, [neg_infinity, 0.0, infinity]}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_float32_errors() -> + [{userdata, [{doc, "Tests vector float32 error handling."}]}]. +vector_float32_errors(_Config) -> + %% FLOAT32 with padding (invalid BSON) + %% BSON hex: 1C00000005766563746F72000A0000000927030000FE420000E04000 (padding=3) + InvalidPaddingBin = + <<28, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 10, 0, 0, 0, 9, 39, 3, 0, 0, 254, 66, 0, + 0, 224, 64, 0>>, + {error, {invalid_vector_float32_padding, 3}} = nbson:decode(InvalidPaddingBin), + %% Insufficient vector data (3 bytes, not multiple of 4) + %% BSON hex: 1700000005766563746F7200050000000927002A2A2A00 + InvalidLengthBin = + <<23, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 5, 0, 0, 0, 9, 39, 0, 42, 42, 42, 0>>, + {error, {invalid_vector_float32_length, 3}} = nbson:decode(InvalidLengthBin). + +vector_packed_bit() -> + [{userdata, [{doc, "Tests vector packed_bit data type BSON encoding/decoding."}]}]. +vector_packed_bit(_Config) -> + %% Test case from spec: Simple Vector PACKED_BIT [127, 7] + %% BSON hex: 1600000005766563746F7200040000000910007F0700 + BaseBin = + <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 16, 0, 127, 7, 0>>, + BaseMap = #{<<"vector">> => {vector, packed_bit, <<127, 7>>}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_packed_bit_with_padding() -> + [{userdata, [{doc, "Tests vector packed_bit with padding BSON encoding/decoding."}]}]. +vector_packed_bit_with_padding(_Config) -> + %% Test case from spec: PACKED_BIT with padding [127, 8], padding=3 + %% BSON hex: 1600000005766563746F7200040000000910037F0800 + BaseBin = + <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 16, 3, 127, 8, 0>>, + BaseMap = #{<<"vector">> => {vector, packed_bit, <<127, 8>>, 3}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_packed_bit_empty() -> + [{userdata, [{doc, "Tests empty vector packed_bit BSON encoding/decoding."}]}]. +vector_packed_bit_empty(_Config) -> + %% Test case from spec: Empty Vector PACKED_BIT [] + %% BSON hex: 1400000005766563746F72000200000009100000 + BaseBin = <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 16, 0, 0>>, + BaseMap = #{<<"vector">> => {vector, packed_bit, <<>>}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_packed_bit_errors() -> + [{userdata, [{doc, "Tests vector packed_bit error handling."}]}]. +vector_packed_bit_errors(_Config) -> + %% Empty with non-zero padding (encode error) + {error, {invalid_vector_packed_bit_empty_with_padding, 1}} = + nbson:encode(#{<<"vector">> => {vector, packed_bit, <<>>, 1}}), + %% Empty with non-zero padding (decode error) + %% BSON hex: 1400000005766563746F72000200000009100100 (padding=1, empty data) + EmptyPaddingBin = + <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 16, 1, 0>>, + {error, {invalid_vector_packed_bit_empty_with_padding, 1}} = nbson:decode(EmptyPaddingBin), + %% Padding exceeds maximum (8) + %% BSON hex: 1500000005766563746F7200030000000910080100 (padding=8) + ExceedPaddingBin = + <<21, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 3, 0, 0, 0, 9, 16, 8, 1, 0>>, + {error, {invalid_vector_padding, 8}} = nbson:decode(ExceedPaddingBin), + %% Invalid dtype + InvalidDtypeBin = + <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 255, 0, 0>>, + {error, {invalid_vector_dtype, 255}} = nbson:decode(InvalidDtypeBin).