From 927c58109d20ed70ea21f95fecfb1a67a9861146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Rubinos=20Rodr=C3=ADguez?= Date: Wed, 17 Dec 2025 17:17:57 +0100 Subject: [PATCH 1/4] feat: binary vector implementation. --- src/nbson.erl | 29 +++++++- src/nbson_decoder.erl | 89 +++++++++++++++++++++++ src/nbson_encoder.erl | 76 ++++++++++++++++++++ test/nbson_SUITE.erl | 164 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 354 insertions(+), 4 deletions(-) diff --git a/src/nbson.erl b/src/nbson.erl index cd30322..164ff11 100644 --- a/src/nbson.erl +++ b/src/nbson.erl @@ -23,6 +23,14 @@ -type document_path() :: [key()]. -type key() :: binary(). -type regex_arg() :: unicode:latin1_chardata() | unicode:chardata() | unicode:external_chardata(). +-type int8() :: -128..127. +-type vector_padding() :: 0..7. +-type float32_value() :: float() | integer() | infinity | neg_infinity. +-type vector() :: + {vector, int8, [int8()]} + | {vector, float32, [float32_value()]} + | {vector, packed_bit, binary()} + | {vector, packed_bit, binary(), vector_padding()}. -type value() :: float() | integer() @@ -44,6 +52,7 @@ | {data, encrypted, binary()} | {data, compressed, binary()} | {data, user, binary()} + | vector() | {object_id, binary()} | erlang:timestamp() | {regex, regex_arg(), regex_arg()} @@ -51,11 +60,20 @@ | {javascript, map(), binary()} | {javascript, document(), binary()} | {timestamp, non_neg_integer(), non_neg_integer()}. --type decode_error_reason() :: invalid_subtype | invalid_bson. +-type vector_error_reason() :: + {invalid_vector_dtype, non_neg_integer()} + | {invalid_vector_padding, integer()} + | {invalid_vector_int8_padding, non_neg_integer()} + | {invalid_vector_float32_padding, non_neg_integer()} + | {invalid_vector_float32_length, non_neg_integer()} + | {invalid_vector_packed_bit_empty_with_padding, non_neg_integer()} + | {invalid_vector_int8_value, integer()}. +-type decode_error_reason() :: invalid_subtype | invalid_bson | vector_error_reason(). -type encode_error_reason() :: {invalid_proplist_document, term()} | {not_unicode_regex, {term(), term()}} - | {integer_too_large, integer()}. + | {integer_too_large, integer()} + | vector_error_reason(). %%% EXPORT TYPES -export_type([ @@ -65,8 +83,13 @@ document_path/0, key/0, value/0, + vector/0, + int8/0, + float32_value/0, + vector_padding/0, decode_error_reason/0, - encode_error_reason/0 + encode_error_reason/0, + vector_error_reason/0 ]). %%%----------------------------------------------------------------------------- diff --git a/src/nbson_decoder.erl b/src/nbson_decoder.erl index 732f39b..6a44049 100644 --- a/src/nbson_decoder.erl +++ b/src/nbson_decoder.erl @@ -246,6 +246,14 @@ string(Base, Len, Next) -> SubType :: non_neg_integer(), Next :: [term()], Result :: nbson:document() | {error, nbson:decode_error_reason()}. +binary(<>, Size, 9, Next) -> + <> = Base, + case decode_vector(Part) of + {error, _} = Error -> + Error; + Vector -> + next(Bin, Vector, Next) + end; binary(<>, Size, SubType, Next) -> <> = Base, case subtype_decode(SubType) of @@ -279,3 +287,84 @@ subtype_decode(128) -> user; subtype_decode(_) -> {error, invalid_subtype}. + +%%%----------------------------------------------------------------------------- +%%% VECTOR DECODING FUNCTIONS +%%%----------------------------------------------------------------------------- +-define(VECTOR_DTYPE_INT8, 16#03). +-define(VECTOR_DTYPE_FLOAT32, 16#27). +-define(VECTOR_DTYPE_PACKED_BIT, 16#10). + +-spec decode_vector(Data) -> Result when + Data :: binary(), + Result :: nbson:vector() | {error, nbson:vector_error_reason()}. +decode_vector(<>) -> + case DType of + ?VECTOR_DTYPE_INT8 -> + decode_vector_int8(VectorData, Padding); + ?VECTOR_DTYPE_FLOAT32 -> + decode_vector_float32(VectorData, Padding); + ?VECTOR_DTYPE_PACKED_BIT -> + decode_vector_packed_bit(VectorData, Padding); + _ -> + {error, {invalid_vector_dtype, DType}} + end. + +-spec decode_vector_int8(Data, Padding) -> Result when + Data :: binary(), + Padding :: non_neg_integer(), + Result :: nbson:vector() | {error, nbson:vector_error_reason()}. +decode_vector_int8(_Data, Padding) when Padding =/= 0 -> + {error, {invalid_vector_int8_padding, Padding}}; +decode_vector_int8(Data, 0) -> + Values = decode_int8_values(Data, []), + {vector, int8, Values}. + +-spec decode_int8_values(Data, Acc) -> Result when + Data :: binary(), + Acc :: [integer()], + Result :: [integer()]. +decode_int8_values(<<>>, Acc) -> + lists:reverse(Acc); +decode_int8_values(<>, Acc) -> + decode_int8_values(Rest, [V | Acc]). + +-spec decode_vector_float32(Data, Padding) -> Result when + Data :: binary(), + Padding :: non_neg_integer(), + Result :: nbson:vector() | {error, nbson:vector_error_reason()}. +decode_vector_float32(_Data, Padding) when Padding =/= 0 -> + {error, {invalid_vector_float32_padding, Padding}}; +decode_vector_float32(Data, 0) when byte_size(Data) rem 4 =/= 0 -> + {error, {invalid_vector_float32_length, byte_size(Data)}}; +decode_vector_float32(Data, 0) -> + Values = decode_float32_values(Data, []), + {vector, float32, Values}. + +-spec decode_float32_values(Data, Acc) -> Result when + Data :: binary(), + Acc :: [nbson:float32_value()], + Result :: [nbson:float32_value()]. +decode_float32_values(<<>>, Acc) -> + lists:reverse(Acc); +decode_float32_values(<<0, 0, 128, 127, Rest/binary>>, Acc) -> + %% Positive infinity (IEEE 754) + decode_float32_values(Rest, [infinity | Acc]); +decode_float32_values(<<0, 0, 128, 255, Rest/binary>>, Acc) -> + %% Negative infinity (IEEE 754) + decode_float32_values(Rest, [neg_infinity | Acc]); +decode_float32_values(<>, Acc) -> + decode_float32_values(Rest, [V | Acc]). + +-spec decode_vector_packed_bit(Data, Padding) -> Result when + Data :: binary(), + Padding :: non_neg_integer(), + Result :: nbson:vector() | {error, nbson:vector_error_reason()}. +decode_vector_packed_bit(_Data, Padding) when Padding > 7 -> + {error, {invalid_vector_padding, Padding}}; +decode_vector_packed_bit(<<>>, Padding) when Padding > 0 -> + {error, {invalid_vector_packed_bit_empty_with_padding, Padding}}; +decode_vector_packed_bit(Data, 0) -> + {vector, packed_bit, Data}; +decode_vector_packed_bit(Data, Padding) -> + {vector, packed_bit, Data, Padding}. diff --git a/src/nbson_encoder.erl b/src/nbson_encoder.erl index 961aff1..17c9548 100644 --- a/src/nbson_encoder.erl +++ b/src/nbson_encoder.erl @@ -158,6 +158,16 @@ encode_value({data, compressed, Data}) when is_binary(Data) -> {?BIN_TYPE, <>}; encode_value({data, user, Data}) when is_binary(Data) -> {?BIN_TYPE, <>}; +encode_value({vector, int8, Values}) when is_list(Values) -> + encode_vector_int8(Values); +encode_value({vector, float32, Values}) when is_list(Values) -> + encode_vector_float32(Values); +encode_value({vector, packed_bit, Data}) when is_binary(Data) -> + encode_vector_packed_bit(Data, 0); +encode_value({vector, packed_bit, Data, Padding}) when + is_binary(Data), Padding >= 0, Padding =< 7 +-> + encode_vector_packed_bit(Data, Padding); encode_value(undefined) -> {?UNDEF_TYPE, <<>>}; encode_value({object_id, <<_:96>> = Id}) -> @@ -254,3 +264,69 @@ map_fold_encode(Label, Value, Acc) -> {Type, Payload} -> <> end. + +%%%----------------------------------------------------------------------------- +%%% VECTOR ENCODING FUNCTIONS +%%%----------------------------------------------------------------------------- +-define(VECTOR_SUBTYPE, 9). +-define(VECTOR_DTYPE_INT8, 16#03). +-define(VECTOR_DTYPE_FLOAT32, 16#27). +-define(VECTOR_DTYPE_PACKED_BIT, 16#10). + +-spec encode_vector_int8(Values) -> Result when + Values :: [integer()], + Result :: {?BIN_TYPE, binary()} | {error, nbson:encode_error_reason()}. +encode_vector_int8(Values) -> + case encode_int8_values(Values, <<>>) of + {error, _Reason} = Error -> + Error; + Data -> + VectorData = <>, + {?BIN_TYPE, << + ?INT32(byte_size(VectorData)), ?INT8(?VECTOR_SUBTYPE), VectorData/binary + >>} + end. + +-spec encode_int8_values(Values, Acc) -> Result when + Values :: [integer()], + Acc :: binary(), + Result :: binary() | {error, nbson:encode_error_reason()}. +encode_int8_values([], Acc) -> + Acc; +encode_int8_values([V | Rest], Acc) when is_integer(V), V >= -128, V =< 127 -> + encode_int8_values(Rest, <>); +encode_int8_values([V | _Rest], _Acc) -> + {error, {invalid_vector_int8_value, V}}. + +-spec encode_vector_float32(Values) -> Result when + Values :: [nbson:float32_value()], + Result :: {?BIN_TYPE, binary()}. +encode_vector_float32(Values) -> + Data = encode_float32_values(Values, <<>>), + VectorData = <>, + {?BIN_TYPE, <>}. + +-spec encode_float32_values(Values, Acc) -> Result when + Values :: [nbson:float32_value()], + Acc :: binary(), + Result :: binary(). +encode_float32_values([], Acc) -> + Acc; +encode_float32_values([infinity | Rest], Acc) -> + %% Positive infinity (IEEE 754 single precision) + encode_float32_values(Rest, <>); +encode_float32_values([neg_infinity | Rest], Acc) -> + %% Negative infinity (IEEE 754 single precision) + encode_float32_values(Rest, <>); +encode_float32_values([V | Rest], Acc) when is_number(V) -> + encode_float32_values(Rest, <>). + +-spec encode_vector_packed_bit(Data, Padding) -> Result when + Data :: binary(), + Padding :: nbson:vector_padding(), + Result :: {?BIN_TYPE, binary()} | {error, nbson:encode_error_reason()}. +encode_vector_packed_bit(<<>>, Padding) when Padding > 0 -> + {error, {invalid_vector_packed_bit_empty_with_padding, Padding}}; +encode_vector_packed_bit(Data, Padding) -> + VectorData = <>, + {?BIN_TYPE, <>}. diff --git a/test/nbson_SUITE.erl b/test/nbson_SUITE.erl index ebddc69..f7d4d89 100644 --- a/test/nbson_SUITE.erl +++ b/test/nbson_SUITE.erl @@ -46,7 +46,18 @@ all() -> timestamp, undef, proplists, - various + various, + vector_int8, + vector_int8_empty, + vector_int8_errors, + vector_float32, + vector_float32_empty, + vector_float32_infinity, + vector_float32_errors, + vector_packed_bit, + vector_packed_bit_with_padding, + vector_packed_bit_empty, + vector_packed_bit_errors ]. %%%----------------------------------------------------------------------------- @@ -362,3 +373,154 @@ various(_Config) -> undefined = nbson:get([<<"arr">>, <<"three">>], #{<<"arr">> => #{<<"two">> => <<"three">>}}), <<"three">> = nbson:get([<<"arr">>, <<"two">>], [{<<"arr">>, [{<<"two">>, <<"three">>}]}]), undefined = nbson:get([<<"arr">>, <<"three">>], [{<<"arr">>, [{<<"two">>, <<"three">>}]}]). + +%%%----------------------------------------------------------------------------- +%%% VECTOR TEST CASES +%%%----------------------------------------------------------------------------- +vector_int8() -> + [{userdata, [{doc, "Tests vector int8 data type BSON encoding/decoding."}]}]. +vector_int8(_Config) -> + %% Test case from spec: Simple Vector INT8 [127, 7] + %% BSON hex: 1600000005766563746F7200040000000903007F0700 + BaseBin = <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 3, 0, 127, 7, 0>>, + BaseMap = #{<<"vector">> => {vector, int8, [127, 7]}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap), + %% Test negative values + NegMap = #{<<"vector">> => {vector, int8, [-128, -1, 0, 1, 127]}}, + {ok, NegBin} = nbson:encode(NegMap), + {ok, NegMap} = nbson:decode(NegBin). + +vector_int8_empty() -> + [{userdata, [{doc, "Tests empty vector int8 data type BSON encoding/decoding."}]}]. +vector_int8_empty(_Config) -> + %% Test case from spec: Empty Vector INT8 [] + %% BSON hex: 1400000005766563746F72000200000009030000 + BaseBin = <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 3, 0, 0>>, + BaseMap = #{<<"vector">> => {vector, int8, []}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_int8_errors() -> + [{userdata, [{doc, "Tests vector int8 error handling."}]}]. +vector_int8_errors(_Config) -> + %% INT8 value overflow (128 is out of range) + {error, {invalid_vector_int8_value, 128}} = + nbson:encode(#{<<"vector">> => {vector, int8, [128]}}), + %% INT8 value underflow (-129 is out of range) + {error, {invalid_vector_int8_value, -129}} = + nbson:encode(#{<<"vector">> => {vector, int8, [-129]}}), + %% INT8 with padding (invalid BSON) + %% BSON hex: 1600000005766563746F7200040000000903037F0700 (padding=3) + InvalidPaddingBin = + <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 3, 3, 127, 7, 0>>, + {error, {invalid_vector_int8_padding, 3}} = nbson:decode(InvalidPaddingBin). + +vector_float32() -> + [{userdata, [{doc, "Tests vector float32 data type BSON encoding/decoding."}]}]. +vector_float32(_Config) -> + %% Test case from spec: Simple Vector FLOAT32 [127.0, 7.0] + %% BSON hex: 1C00000005766563746F72000A0000000927000000FE420000E04000 + BaseBin = + <<28, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 10, 0, 0, 0, 9, 39, 0, 0, 0, 254, 66, 0, + 0, 224, 64, 0>>, + BaseMap = #{<<"vector">> => {vector, float32, [127.0, 7.0]}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap), + %% Test with decimals and negative value: [127.7, -7.7] + %% BSON hex: 1C00000005766563746F72000A0000000927006666FF426666F6C000 + DecBin = + <<28, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 10, 0, 0, 0, 9, 39, 0, 102, 102, 255, 66, + 102, 102, 246, 192, 0>>, + DecMap = #{<<"vector">> => {vector, float32, [127.69999694824219, -7.699999809265137]}}, + {ok, DecMap} = nbson:decode(DecBin). + +vector_float32_empty() -> + [{userdata, [{doc, "Tests empty vector float32 data type BSON encoding/decoding."}]}]. +vector_float32_empty(_Config) -> + %% Test case from spec: Empty Vector FLOAT32 [] + %% BSON hex: 1400000005766563746F72000200000009270000 + BaseBin = <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 39, 0, 0>>, + BaseMap = #{<<"vector">> => {vector, float32, []}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_float32_infinity() -> + [{userdata, [{doc, "Tests vector float32 with infinity values."}]}]. +vector_float32_infinity(_Config) -> + %% Test case from spec: Infinity Vector FLOAT32 [-Infinity, 0.0, Infinity] + %% BSON hex: 2000000005766563746F72000E000000092700000080FF000000000000807F00 + BaseBin = + <<32, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 14, 0, 0, 0, 9, 39, 0, 0, 0, 128, 255, 0, + 0, 0, 0, 0, 0, 128, 127, 0>>, + BaseMap = #{<<"vector">> => {vector, float32, [neg_infinity, 0.0, infinity]}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_float32_errors() -> + [{userdata, [{doc, "Tests vector float32 error handling."}]}]. +vector_float32_errors(_Config) -> + %% FLOAT32 with padding (invalid BSON) + %% BSON hex: 1C00000005766563746F72000A0000000927030000FE420000E04000 (padding=3) + InvalidPaddingBin = + <<28, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 10, 0, 0, 0, 9, 39, 3, 0, 0, 254, 66, 0, + 0, 224, 64, 0>>, + {error, {invalid_vector_float32_padding, 3}} = nbson:decode(InvalidPaddingBin), + %% Insufficient vector data (3 bytes, not multiple of 4) + %% BSON hex: 1700000005766563746F7200050000000927002A2A2A00 + InvalidLengthBin = + <<23, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 5, 0, 0, 0, 9, 39, 0, 42, 42, 42, 0>>, + {error, {invalid_vector_float32_length, 3}} = nbson:decode(InvalidLengthBin). + +vector_packed_bit() -> + [{userdata, [{doc, "Tests vector packed_bit data type BSON encoding/decoding."}]}]. +vector_packed_bit(_Config) -> + %% Test case from spec: Simple Vector PACKED_BIT [127, 7] + %% BSON hex: 1600000005766563746F7200040000000910007F0700 + BaseBin = + <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 16, 0, 127, 7, 0>>, + BaseMap = #{<<"vector">> => {vector, packed_bit, <<127, 7>>}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_packed_bit_with_padding() -> + [{userdata, [{doc, "Tests vector packed_bit with padding BSON encoding/decoding."}]}]. +vector_packed_bit_with_padding(_Config) -> + %% Test case from spec: PACKED_BIT with padding [127, 8], padding=3 + %% BSON hex: 1600000005766563746F7200040000000910037F0800 + BaseBin = + <<22, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 4, 0, 0, 0, 9, 16, 3, 127, 8, 0>>, + BaseMap = #{<<"vector">> => {vector, packed_bit, <<127, 8>>, 3}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_packed_bit_empty() -> + [{userdata, [{doc, "Tests empty vector packed_bit BSON encoding/decoding."}]}]. +vector_packed_bit_empty(_Config) -> + %% Test case from spec: Empty Vector PACKED_BIT [] + %% BSON hex: 1400000005766563746F72000200000009100000 + BaseBin = <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 16, 0, 0>>, + BaseMap = #{<<"vector">> => {vector, packed_bit, <<>>}}, + {ok, BaseMap} = nbson:decode(BaseBin), + {ok, BaseBin} = nbson:encode(BaseMap). + +vector_packed_bit_errors() -> + [{userdata, [{doc, "Tests vector packed_bit error handling."}]}]. +vector_packed_bit_errors(_Config) -> + %% Empty with non-zero padding (encode error) + {error, {invalid_vector_packed_bit_empty_with_padding, 1}} = + nbson:encode(#{<<"vector">> => {vector, packed_bit, <<>>, 1}}), + %% Empty with non-zero padding (decode error) + %% BSON hex: 1400000005766563746F72000200000009100100 (padding=1, empty data) + EmptyPaddingBin = + <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 16, 1, 0>>, + {error, {invalid_vector_packed_bit_empty_with_padding, 1}} = nbson:decode(EmptyPaddingBin), + %% Padding exceeds maximum (8) + %% BSON hex: 1500000005766563746F7200030000000910080100 (padding=8) + ExceedPaddingBin = + <<21, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 3, 0, 0, 0, 9, 16, 8, 1, 0>>, + {error, {invalid_vector_padding, 8}} = nbson:decode(ExceedPaddingBin), + %% Invalid dtype + InvalidDtypeBin = + <<20, 0, 0, 0, 5, 118, 101, 99, 116, 111, 114, 0, 2, 0, 0, 0, 9, 255, 0, 0>>, + {error, {invalid_vector_dtype, 255}} = nbson:decode(InvalidDtypeBin). From fbf3475cbfa3e24812b5f959a8cbf4be63efcf8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Rubinos=20Rodr=C3=ADguez?= Date: Wed, 17 Dec 2025 17:21:57 +0100 Subject: [PATCH 2/4] forgot update README.md --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index fb7a4bd..cbd37b3 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ The following table represents the association between Erlang types and BSON typ | `[ tuple() \| _ ] ` | Object (3) | | `list()` | Array (4) | | `{data, binary, binary()}` | Binary data (5) | +| `{vector, int8, [integer()]}` | Binary Vector INT8 (5, subtype 9) | +| `{vector, float32, [float()]}` | Binary Vector FLOAT32 (5, subtype 9) | +| `{vector, packed_bit, binary()}` | Binary Vector PACKED_BIT (5, subtype 9) | +| `{vector, packed_bit, binary(), 0..7}` | Binary Vector PACKED_BIT with padding (5, subtype 9) | | `undefined` | Undefined (6) | | `{object_id, <<_:96>>}` | ObjectId (7) | | `false \| true` | Boolean (8) | @@ -53,6 +57,29 @@ The following table represents the association between Erlang types and BSON typ On deserialization, we prevent the dynamic generation of atoms by converting BSON Symbol, Max Key and Min Key values to Erlang binaries. +### Binary Vectors + +`nbson` supports [BSON Binary Subtype 9](https://github.com/mongodb/specifications/blob/master/source/bson-binary-vector/bson-binary-vector.md) for efficient storage of dense numeric vectors, useful for AI/ML applications and semantic search. + +| Vector Type | Erlang Representation | Value Range | +| ----------- | --------------------- | ----------- | +| INT8 | `{vector, int8, [integer()]}` | -128 to 127 | +| FLOAT32 | `{vector, float32, [float()]}` | IEEE 754 single-precision (supports `infinity`, `neg_infinity`) | +| PACKED_BIT | `{vector, packed_bit, binary()}` | Raw bytes (0-255) representing packed bits | +| PACKED_BIT | `{vector, packed_bit, binary(), Padding}` | With padding (0-7 bits to ignore in final byte) | + +```erlang +%% INT8 vector +{ok, Bin} = nbson:encode(#{<<"embedding">> => {vector, int8, [127, -128, 0, 42]}}). +{ok, #{<<"embedding">> := {vector, int8, [127, -128, 0, 42]}}} = nbson:decode(Bin). + +%% FLOAT32 vector with special values +{ok, Bin} = nbson:encode(#{<<"v">> => {vector, float32, [1.5, infinity, neg_infinity]}}). + +%% PACKED_BIT vector with padding (13 bits = 2 bytes with 3 bits padding) +{ok, Bin} = nbson:encode(#{<<"bits">> => {vector, packed_bit, <<255, 248>>, 3}}). +``` + ## Benchmarking The BSON decoder implementation in `nbson_decoder.erl` uses [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style). In this particular case, CPS leads to the use of the [sub binary delayed optimization](https://www.erlang.org/doc/efficiency_guide/binaryhandling.html#match-context) and improved efficiency in the deserialization process. From 37515de25f121e54f321c6b36e87e41688facbca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Rubinos=20Rodr=C3=ADguez?= Date: Wed, 17 Dec 2025 17:31:02 +0100 Subject: [PATCH 3/4] fix: bin_opt_info warned about three unoptimized binary handling. --- src/nbson.erl | 2 +- src/nbson_decoder.erl | 53 ++++++++++++------------------------------- 2 files changed, 16 insertions(+), 39 deletions(-) diff --git a/src/nbson.erl b/src/nbson.erl index 164ff11..1c63c42 100644 --- a/src/nbson.erl +++ b/src/nbson.erl @@ -27,7 +27,7 @@ -type vector_padding() :: 0..7. -type float32_value() :: float() | integer() | infinity | neg_infinity. -type vector() :: - {vector, int8, [int8()]} + {vector, int8, [integer()]} | {vector, float32, [float32_value()]} | {vector, packed_bit, binary()} | {vector, packed_bit, binary(), vector_padding()}. diff --git a/src/nbson_decoder.erl b/src/nbson_decoder.erl index 6a44049..fc02c7a 100644 --- a/src/nbson_decoder.erl +++ b/src/nbson_decoder.erl @@ -300,26 +300,28 @@ subtype_decode(_) -> Result :: nbson:vector() | {error, nbson:vector_error_reason()}. decode_vector(<>) -> case DType of + ?VECTOR_DTYPE_INT8 when Padding =:= 0 -> + {vector, int8, decode_int8_values(VectorData, [])}; ?VECTOR_DTYPE_INT8 -> - decode_vector_int8(VectorData, Padding); + {error, {invalid_vector_int8_padding, Padding}}; + ?VECTOR_DTYPE_FLOAT32 when Padding =:= 0, byte_size(VectorData) rem 4 =:= 0 -> + {vector, float32, decode_float32_values(VectorData, [])}; + ?VECTOR_DTYPE_FLOAT32 when Padding =:= 0 -> + {error, {invalid_vector_float32_length, byte_size(VectorData)}}; ?VECTOR_DTYPE_FLOAT32 -> - decode_vector_float32(VectorData, Padding); + {error, {invalid_vector_float32_padding, Padding}}; + ?VECTOR_DTYPE_PACKED_BIT when Padding > 7 -> + {error, {invalid_vector_padding, Padding}}; + ?VECTOR_DTYPE_PACKED_BIT when Padding > 0, byte_size(VectorData) =:= 0 -> + {error, {invalid_vector_packed_bit_empty_with_padding, Padding}}; + ?VECTOR_DTYPE_PACKED_BIT when Padding =:= 0 -> + {vector, packed_bit, VectorData}; ?VECTOR_DTYPE_PACKED_BIT -> - decode_vector_packed_bit(VectorData, Padding); + {vector, packed_bit, VectorData, Padding}; _ -> {error, {invalid_vector_dtype, DType}} end. --spec decode_vector_int8(Data, Padding) -> Result when - Data :: binary(), - Padding :: non_neg_integer(), - Result :: nbson:vector() | {error, nbson:vector_error_reason()}. -decode_vector_int8(_Data, Padding) when Padding =/= 0 -> - {error, {invalid_vector_int8_padding, Padding}}; -decode_vector_int8(Data, 0) -> - Values = decode_int8_values(Data, []), - {vector, int8, Values}. - -spec decode_int8_values(Data, Acc) -> Result when Data :: binary(), Acc :: [integer()], @@ -329,18 +331,6 @@ decode_int8_values(<<>>, Acc) -> decode_int8_values(<>, Acc) -> decode_int8_values(Rest, [V | Acc]). --spec decode_vector_float32(Data, Padding) -> Result when - Data :: binary(), - Padding :: non_neg_integer(), - Result :: nbson:vector() | {error, nbson:vector_error_reason()}. -decode_vector_float32(_Data, Padding) when Padding =/= 0 -> - {error, {invalid_vector_float32_padding, Padding}}; -decode_vector_float32(Data, 0) when byte_size(Data) rem 4 =/= 0 -> - {error, {invalid_vector_float32_length, byte_size(Data)}}; -decode_vector_float32(Data, 0) -> - Values = decode_float32_values(Data, []), - {vector, float32, Values}. - -spec decode_float32_values(Data, Acc) -> Result when Data :: binary(), Acc :: [nbson:float32_value()], @@ -355,16 +345,3 @@ decode_float32_values(<<0, 0, 128, 255, Rest/binary>>, Acc) -> decode_float32_values(Rest, [neg_infinity | Acc]); decode_float32_values(<>, Acc) -> decode_float32_values(Rest, [V | Acc]). - --spec decode_vector_packed_bit(Data, Padding) -> Result when - Data :: binary(), - Padding :: non_neg_integer(), - Result :: nbson:vector() | {error, nbson:vector_error_reason()}. -decode_vector_packed_bit(_Data, Padding) when Padding > 7 -> - {error, {invalid_vector_padding, Padding}}; -decode_vector_packed_bit(<<>>, Padding) when Padding > 0 -> - {error, {invalid_vector_packed_bit_empty_with_padding, Padding}}; -decode_vector_packed_bit(Data, 0) -> - {vector, packed_bit, Data}; -decode_vector_packed_bit(Data, Padding) -> - {vector, packed_bit, Data, Padding}. From 571629f3a35f286587ed717fe70df3837872c80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Rubinos=20Rodr=C3=ADguez?= Date: Thu, 18 Dec 2025 09:44:13 +0100 Subject: [PATCH 4/4] improved encode_int8_values per Marce recommendation. --- src/nbson_encoder.erl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/nbson_encoder.erl b/src/nbson_encoder.erl index 17c9548..9e7074a 100644 --- a/src/nbson_encoder.erl +++ b/src/nbson_encoder.erl @@ -277,7 +277,7 @@ map_fold_encode(Label, Value, Acc) -> Values :: [integer()], Result :: {?BIN_TYPE, binary()} | {error, nbson:encode_error_reason()}. encode_vector_int8(Values) -> - case encode_int8_values(Values, <<>>) of + case encode_int8_values(Values) of {error, _Reason} = Error -> Error; Data -> @@ -287,16 +287,18 @@ encode_vector_int8(Values) -> >>} end. --spec encode_int8_values(Values, Acc) -> Result when +-spec encode_int8_values(Values) -> Result when Values :: [integer()], - Acc :: binary(), Result :: binary() | {error, nbson:encode_error_reason()}. -encode_int8_values([], Acc) -> - Acc; -encode_int8_values([V | Rest], Acc) when is_integer(V), V >= -128, V =< 127 -> - encode_int8_values(Rest, <>); -encode_int8_values([V | _Rest], _Acc) -> - {error, {invalid_vector_int8_value, V}}. +encode_int8_values(Values) -> + Data = <<<> || V <- Values, V >= -128, V =< 127>>, + case byte_size(Data) == length(Values) of + true -> + Data; + false -> + {value, InvalidValue} = lists:search(fun(V) -> V < -128 orelse V > 127 end, Values), + {error, {invalid_vector_int8_value, InvalidValue}} + end. -spec encode_vector_float32(Values) -> Result when Values :: [nbson:float32_value()],