From 4bb694c34f41cd2ce9d17836cce9e0dadb1a6f46 Mon Sep 17 00:00:00 2001 From: taiyang-li Date: Mon, 13 Jul 2026 16:42:40 +0800 Subject: [PATCH 1/2] feat(sparksql): support decimal format_number Add decimal overloads for format_number with Spark-style grouping and HALF_EVEN rounding. Cover short and long decimal formatting, rounding, and negative decimal-place behavior with tests. --- bolt/functions/sparksql/FormatNumber.cpp | 207 ++++++++++++++++++ .../sparksql/registration/RegisterString.cpp | 1 + .../sparksql/tests/FormatNumberTest.cpp | 54 +++++ 3 files changed, 262 insertions(+) diff --git a/bolt/functions/sparksql/FormatNumber.cpp b/bolt/functions/sparksql/FormatNumber.cpp index aab2789da..e9f5fd280 100644 --- a/bolt/functions/sparksql/FormatNumber.cpp +++ b/bolt/functions/sparksql/FormatNumber.cpp @@ -16,11 +16,15 @@ #include "bolt/functions/sparksql/FormatNumber.h" +#include "bolt/expression/DecodedArgs.h" +#include "bolt/expression/VectorFunction.h" +#include "bolt/type/HugeInt.h" #include #include #include #include #include +#include namespace bytedance::bolt::functions::sparksql::detail { @@ -49,6 +53,129 @@ const std::locale& usLocale() { return loc; } +void appendGroupedIntegerDigits( + std::string_view digits, + exec::StringWriter& out) { + if (digits.empty()) { + out.append(std::string_view("0")); + return; + } + + const auto firstGroupSize = + digits.size() <= 3 ? digits.size() : ((digits.size() - 1) % 3) + 1; + out.append(digits.substr(0, firstGroupSize)); + + for (size_t offset = firstGroupSize; offset < digits.size(); offset += 3) { + out.append(std::string_view(",")); + out.append(digits.substr(offset, 3)); + } +} + +template +std::string toUnsignedDigits(T value) { + std::ostringstream out; + if constexpr (sizeof(T) < sizeof(int128_t)) { + const int128_t widened = value; + out << (widened < 0 ? -widened : widened); + } else { + out << (value < 0 ? -value : value); + } + return out.str(); +} + +bool shouldRoundHalfEven( + std::string_view keptDigits, + std::string_view discardedDigits) { + if (discardedDigits.empty()) { + return false; + } + + if (discardedDigits[0] > '5') { + return true; + } + if (discardedDigits[0] < '5') { + return false; + } + if (discardedDigits.find_first_not_of('0', 1) != std::string_view::npos) { + return true; + } + + const char lastKeptDigit = keptDigits.empty() ? '0' : keptDigits.back(); + return ((lastKeptDigit - '0') % 2) == 1; +} + +void incrementDigits(std::string& digits) { + for (auto it = digits.rbegin(); it != digits.rend(); ++it) { + if (*it != '9') { + ++(*it); + return; + } + *it = '0'; + } + digits.insert(digits.begin(), '1'); +} + +std::pair splitDecimalDigits( + std::string_view digits, + int32_t scale) { + if (scale == 0) { + return {std::string(digits), ""}; + } + + if (digits.size() <= scale) { + return { + "0", + std::string(scale - digits.size(), '0').append(digits.begin(), digits.end())}; + } + + return { + std::string(digits.substr(0, digits.size() - scale)), + std::string(digits.substr(digits.size() - scale))}; +} + +template +void formatDecimal( + T value, + int32_t inputScale, + int32_t decimalPlaces, + exec::StringWriter& out) { + const int32_t cappedPlaces = std::min(decimalPlaces, kMaxFractionDigits); + const bool negative = value < 0; + const std::string digits = toUnsignedDigits(value); + + std::string roundedDigits; + if (cappedPlaces >= inputScale) { + roundedDigits = digits; + } else { + const int32_t discardedCount = inputScale - cappedPlaces; + const size_t split = digits.size() > discardedCount + ? digits.size() - discardedCount + : 0; + + roundedDigits = split == 0 ? "0" : std::string(digits.substr(0, split)); + const std::string_view discardedDigits(digits.data() + split, digits.size() - split); + if (shouldRoundHalfEven(roundedDigits, discardedDigits)) { + incrementDigits(roundedDigits); + } + } + + auto [integerDigits, fractionDigits] = splitDecimalDigits( + roundedDigits, cappedPlaces < inputScale ? cappedPlaces : inputScale); + + if (cappedPlaces > inputScale) { + fractionDigits.append(cappedPlaces - inputScale, '0'); + } + + if (negative) { + out.append(std::string_view("-")); + } + appendGroupedIntegerDigits(integerDigits, out); + if (cappedPlaces > 0) { + out.append(std::string_view(".")); + out.append(fractionDigits); + } +} + } // namespace void formatInteger( @@ -100,4 +227,84 @@ void formatFloatingPoint( out.append(std::string_view(buf.data(), buf.size())); } +namespace { + +template +class DecimalFormatNumberFunction final : public exec::VectorFunction { + public: + explicit DecimalFormatNumberFunction(int32_t inputScale) + : inputScale_(inputScale) {} + + void apply( + const SelectivityVector& rows, + std::vector& args, + const TypePtr& outputType, + exec::EvalCtx& context, + VectorPtr& result) const override { + exec::DecodedArgs decodedArgs(rows, args, context); + auto* decimalValues = decodedArgs.at(0); + auto* decimalPlaces = decodedArgs.at(1); + + context.ensureWritable(rows, outputType, result); + result->clearNulls(rows); + auto* flatResult = result->asFlatVector(); + flatResult->getBufferWithSpace(rows.countSelected() * 392); + + context.applyToSelectedNoThrow(rows, [&](vector_size_t row) { + const auto places = decimalPlaces->valueAt(row); + if (places < 0) { + result->setNull(row, true); + return; + } + exec::StringWriter writer(flatResult, row); + formatDecimal( + decimalValues->valueAt(row), + inputScale_, + places, + writer); + writer.finalize(); + }); + } + + private: + const int32_t inputScale_; +}; + +std::vector> +formatNumberDecimalSignatures() { + return {exec::FunctionSignatureBuilder() + .integerVariable("a_precision") + .integerVariable("a_scale") + .returnType("varchar") + .argumentType("DECIMAL(a_precision, a_scale)") + .argumentType("integer") + .build()}; +} + +std::shared_ptr createDecimalFormatNumberFunction( + const std::string& /*name*/, + const std::vector& inputArgs, + const core::QueryConfig& /*config*/) { + BOLT_CHECK_EQ(inputArgs.size(), 2); + const auto& decimalType = inputArgs[0].type; + if (decimalType->isShortDecimal()) { + return std::make_shared>( + decimalType->asShortDecimal().scale()); + } + BOLT_USER_CHECK(decimalType->isLongDecimal(), "Expect decimal input type."); + return std::make_shared>( + decimalType->asLongDecimal().scale()); +} + +} // namespace + } // namespace bytedance::bolt::functions::sparksql::detail + +namespace bytedance::bolt::functions::sparksql { + +BOLT_DECLARE_STATEFUL_VECTOR_FUNCTION( + udf_decimal_format_number, + detail::formatNumberDecimalSignatures(), + detail::createDecimalFormatNumberFunction); + +} // namespace bytedance::bolt::functions::sparksql diff --git a/bolt/functions/sparksql/registration/RegisterString.cpp b/bolt/functions/sparksql/registration/RegisterString.cpp index f4a9e9cff..afe30710f 100644 --- a/bolt/functions/sparksql/registration/RegisterString.cpp +++ b/bolt/functions/sparksql/registration/RegisterString.cpp @@ -70,6 +70,7 @@ void registerStringFunctions(const std::string& prefix) { registerFunction( {prefix + "to_title", prefix + "initcap"}); + BOLT_REGISTER_VECTOR_FUNCTION(udf_decimal_format_number, prefix + "format_number"); registerFunction( {prefix + "format_number"}); registerFunction( diff --git a/bolt/functions/sparksql/tests/FormatNumberTest.cpp b/bolt/functions/sparksql/tests/FormatNumberTest.cpp index 87dbe95b2..9854a8a06 100644 --- a/bolt/functions/sparksql/tests/FormatNumberTest.cpp +++ b/bolt/functions/sparksql/tests/FormatNumberTest.cpp @@ -17,6 +17,7 @@ #include #include "bolt/functions/sparksql/tests/SparkFunctionBaseTest.h" +#include "bolt/type/HugeInt.h" namespace bytedance::bolt::functions::sparksql::test { @@ -45,6 +46,26 @@ class FormatNumberTest : public SparkFunctionBaseTest { std::optional formatFloat(float value, int32_t d) { return formatNumber(value, d); } + + std::optional formatShortDecimal( + int64_t unscaledValue, + const TypePtr& type, + int32_t decimalPlaces) { + return evaluateOnce( + "format_number(c0, c1)", + makeRowVector({makeFlatVector({unscaledValue}, type), + makeFlatVector({decimalPlaces})})); + } + + std::optional formatLongDecimal( + int128_t unscaledValue, + const TypePtr& type, + int32_t decimalPlaces) { + return evaluateOnce( + "format_number(c0, c1)", + makeRowVector({makeFlatVector({unscaledValue}, type), + makeFlatVector({decimalPlaces})})); + } }; TEST_F(FormatNumberTest, integers) { @@ -117,6 +138,38 @@ TEST_F(FormatNumberTest, binaryTieRounding) { EXPECT_EQ(formatDouble(-2.675, 2), "-2.67"); } +TEST_F(FormatNumberTest, decimals) { + EXPECT_EQ(formatShortDecimal(1234567, DECIMAL(10, 2), 2), "12,345.67"); + EXPECT_EQ(formatShortDecimal(1234567, DECIMAL(10, 2), 1), "12,345.7"); + EXPECT_EQ(formatShortDecimal(1234567, DECIMAL(10, 2), 0), "12,346"); + EXPECT_EQ(formatShortDecimal(-1234567, DECIMAL(10, 2), 3), "-12,345.670"); + EXPECT_EQ( + formatShortDecimal(123456789012345678LL, DECIMAL(18, 4), 4), + "12,345,678,901,234.5678"); + + EXPECT_EQ( + formatLongDecimal( + HugeInt::parse("12345678901234567890123456789012345678"), + DECIMAL(38, 4), + 4), + "1,234,567,890,123,456,789,012,345,678,901,234.5678"); + EXPECT_EQ( + formatLongDecimal( + HugeInt::parse("12345678901234567890123456789012345678"), + DECIMAL(38, 20), + 2), + "123,456,789,012,345,678.90"); +} + +TEST_F(FormatNumberTest, decimalHalfEvenRounding) { + EXPECT_EQ(formatShortDecimal(125, DECIMAL(4, 2), 1), "1.2"); + EXPECT_EQ(formatShortDecimal(135, DECIMAL(4, 2), 1), "1.4"); + EXPECT_EQ(formatShortDecimal(250, DECIMAL(4, 2), 0), "2"); + EXPECT_EQ(formatShortDecimal(350, DECIMAL(4, 2), 0), "4"); + EXPECT_EQ(formatShortDecimal(-250, DECIMAL(4, 2), 0), "-2"); + EXPECT_EQ(formatShortDecimal(-350, DECIMAL(4, 2), 0), "-4"); +} + TEST_F(FormatNumberTest, negativeDecimalPlaces) { // d < 0 returns null per Spark semantics. EXPECT_EQ( @@ -127,6 +180,7 @@ TEST_F(FormatNumberTest, negativeDecimalPlaces) { (evaluateOnce( "format_number(c0, c1)", 12345.6, -2)), std::nullopt); + EXPECT_EQ(formatShortDecimal(12345, DECIMAL(10, 2), -1), std::nullopt); } TEST_F(FormatNumberTest, boundaryValues) { From 1ef08d63b99dad3835b0670c736b1370f05a3d51 Mon Sep 17 00:00:00 2001 From: taiyang-li Date: Mon, 13 Jul 2026 17:51:15 +0800 Subject: [PATCH 2/2] fix code style --- bolt/functions/sparksql/FormatNumber.cpp | 174 +++++++++--------- .../sparksql/registration/RegisterString.cpp | 3 +- .../sparksql/tests/FormatNumberTest.cpp | 10 +- 3 files changed, 92 insertions(+), 95 deletions(-) diff --git a/bolt/functions/sparksql/FormatNumber.cpp b/bolt/functions/sparksql/FormatNumber.cpp index e9f5fd280..c5bfee404 100644 --- a/bolt/functions/sparksql/FormatNumber.cpp +++ b/bolt/functions/sparksql/FormatNumber.cpp @@ -16,15 +16,15 @@ #include "bolt/functions/sparksql/FormatNumber.h" -#include "bolt/expression/DecodedArgs.h" -#include "bolt/expression/VectorFunction.h" -#include "bolt/type/HugeInt.h" #include #include +#include #include #include #include -#include +#include "bolt/expression/DecodedArgs.h" +#include "bolt/expression/VectorFunction.h" +#include "bolt/type/DecimalUtil.h" namespace bytedance::bolt::functions::sparksql::detail { @@ -53,6 +53,11 @@ const std::locale& usLocale() { return loc; } +std::string_view zeroPadding(int32_t zeroCount) { + static const std::string kZeros(kMaxFractionDigits, '0'); + return std::string_view(kZeros.data(), zeroCount); +} + void appendGroupedIntegerDigits( std::string_view digits, exec::StringWriter& out) { @@ -72,65 +77,46 @@ void appendGroupedIntegerDigits( } template -std::string toUnsignedDigits(T value) { - std::ostringstream out; - if constexpr (sizeof(T) < sizeof(int128_t)) { - const int128_t widened = value; - out << (widened < 0 ? -widened : widened); - } else { - out << (value < 0 ? -value : value); - } - return out.str(); -} - -bool shouldRoundHalfEven( - std::string_view keptDigits, - std::string_view discardedDigits) { - if (discardedDigits.empty()) { - return false; +T roundDecimalHalfEven(T value, int32_t fromScale, int32_t toScale) { + if (toScale >= fromScale) { + return value; } - if (discardedDigits[0] > '5') { - return true; - } - if (discardedDigits[0] < '5') { - return false; - } - if (discardedDigits.find_first_not_of('0', 1) != std::string_view::npos) { - return true; + const auto scaleFactor = + DecimalUtil::getPowersOfTen(static_cast(fromScale - toScale)); + int128_t roundedValue = value; + int128_t remainder = roundedValue % scaleFactor; + roundedValue /= scaleFactor; + + const int128_t absRemainder = remainder < 0 ? -remainder : remainder; + const int128_t twiceRemainder = absRemainder * 2; + if (twiceRemainder > scaleFactor) { + roundedValue += value >= 0 ? 1 : -1; + } else if ( + twiceRemainder == scaleFactor && + ((roundedValue < 0 ? -roundedValue : roundedValue) % 2 == 1)) { + roundedValue += value >= 0 ? 1 : -1; } - const char lastKeptDigit = keptDigits.empty() ? '0' : keptDigits.back(); - return ((lastKeptDigit - '0') % 2) == 1; + return static_cast(roundedValue); } -void incrementDigits(std::string& digits) { - for (auto it = digits.rbegin(); it != digits.rend(); ++it) { - if (*it != '9') { - ++(*it); - return; - } - *it = '0'; - } - digits.insert(digits.begin(), '1'); -} - -std::pair splitDecimalDigits( - std::string_view digits, - int32_t scale) { - if (scale == 0) { - return {std::string(digits), ""}; - } - - if (digits.size() <= scale) { - return { - "0", - std::string(scale - digits.size(), '0').append(digits.begin(), digits.end())}; +template +size_t estimateFormattedDecimalSize( + T value, + int32_t inputPrecision, + int32_t inputScale, + int32_t decimalPlaces) { + if (decimalPlaces < 0) { + return 0; } - return { - std::string(digits.substr(0, digits.size() - scale)), - std::string(digits.substr(digits.size() - scale))}; + const int32_t cappedPlaces = std::min(decimalPlaces, kMaxFractionDigits); + const int32_t outputScale = std::min(cappedPlaces, inputScale); + const int32_t integerDigits = std::max(inputPrecision - outputScale, 1); + const int32_t groupingSeparators = (integerDigits - 1) / 3; + return (value < 0 ? 1 : 0) + integerDigits + groupingSeparators + + (cappedPlaces > 0 ? 1 + cappedPlaces : 0); } template @@ -140,39 +126,36 @@ void formatDecimal( int32_t decimalPlaces, exec::StringWriter& out) { const int32_t cappedPlaces = std::min(decimalPlaces, kMaxFractionDigits); - const bool negative = value < 0; - const std::string digits = toUnsignedDigits(value); - - std::string roundedDigits; - if (cappedPlaces >= inputScale) { - roundedDigits = digits; - } else { - const int32_t discardedCount = inputScale - cappedPlaces; - const size_t split = digits.size() > discardedCount - ? digits.size() - discardedCount - : 0; - - roundedDigits = split == 0 ? "0" : std::string(digits.substr(0, split)); - const std::string_view discardedDigits(digits.data() + split, digits.size() - split); - if (shouldRoundHalfEven(roundedDigits, discardedDigits)) { - incrementDigits(roundedDigits); - } + const int32_t outputScale = std::min(cappedPlaces, inputScale); + const T roundedValue = roundDecimalHalfEven(value, inputScale, outputScale); + + std::array plainBuffer; + const auto plainSize = DecimalUtil::convertToPlainString( + roundedValue, outputScale, plainBuffer.size(), plainBuffer.data()); + const std::string_view plainNumber(plainBuffer.data(), plainSize); + size_t start = 0; + if (!plainNumber.empty() && plainNumber.front() == '-') { + out.append(std::string_view("-")); + start = 1; } - auto [integerDigits, fractionDigits] = splitDecimalDigits( - roundedDigits, cappedPlaces < inputScale ? cappedPlaces : inputScale); + const auto decimalPoint = plainNumber.find('.', start); + appendGroupedIntegerDigits( + decimalPoint == std::string_view::npos + ? plainNumber.substr(start) + : plainNumber.substr(start, decimalPoint - start), + out); - if (cappedPlaces > inputScale) { - fractionDigits.append(cappedPlaces - inputScale, '0'); + if (decimalPoint != std::string_view::npos) { + out.append(plainNumber.substr(decimalPoint)); } - if (negative) { - out.append(std::string_view("-")); - } - appendGroupedIntegerDigits(integerDigits, out); - if (cappedPlaces > 0) { - out.append(std::string_view(".")); - out.append(fractionDigits); + const auto trailingZeros = cappedPlaces - outputScale; + if (trailingZeros > 0) { + if (decimalPoint == std::string_view::npos) { + out.append(std::string_view(".")); + } + out.append(zeroPadding(trailingZeros)); } } @@ -232,8 +215,8 @@ namespace { template class DecimalFormatNumberFunction final : public exec::VectorFunction { public: - explicit DecimalFormatNumberFunction(int32_t inputScale) - : inputScale_(inputScale) {} + DecimalFormatNumberFunction(int32_t inputPrecision, int32_t inputScale) + : inputPrecision_(inputPrecision), inputScale_(inputScale) {} void apply( const SelectivityVector& rows, @@ -248,7 +231,18 @@ class DecimalFormatNumberFunction final : public exec::VectorFunction { context.ensureWritable(rows, outputType, result); result->clearNulls(rows); auto* flatResult = result->asFlatVector(); - flatResult->getBufferWithSpace(rows.countSelected() * 392); + + size_t estimatedBufferSize = 0; + rows.applyToSelected([&](vector_size_t row) { + estimatedBufferSize += estimateFormattedDecimalSize( + decimalValues->valueAt(row), + inputPrecision_, + inputScale_, + decimalPlaces->valueAt(row)); + }); + if (estimatedBufferSize > 0) { + flatResult->getBufferWithSpace(estimatedBufferSize, true); + } context.applyToSelectedNoThrow(rows, [&](vector_size_t row) { const auto places = decimalPlaces->valueAt(row); @@ -258,15 +252,13 @@ class DecimalFormatNumberFunction final : public exec::VectorFunction { } exec::StringWriter writer(flatResult, row); formatDecimal( - decimalValues->valueAt(row), - inputScale_, - places, - writer); + decimalValues->valueAt(row), inputScale_, places, writer); writer.finalize(); }); } private: + const int32_t inputPrecision_; const int32_t inputScale_; }; @@ -289,10 +281,12 @@ std::shared_ptr createDecimalFormatNumberFunction( const auto& decimalType = inputArgs[0].type; if (decimalType->isShortDecimal()) { return std::make_shared>( + decimalType->asShortDecimal().precision(), decimalType->asShortDecimal().scale()); } BOLT_USER_CHECK(decimalType->isLongDecimal(), "Expect decimal input type."); return std::make_shared>( + decimalType->asLongDecimal().precision(), decimalType->asLongDecimal().scale()); } diff --git a/bolt/functions/sparksql/registration/RegisterString.cpp b/bolt/functions/sparksql/registration/RegisterString.cpp index afe30710f..c8a7e4a2e 100644 --- a/bolt/functions/sparksql/registration/RegisterString.cpp +++ b/bolt/functions/sparksql/registration/RegisterString.cpp @@ -70,7 +70,8 @@ void registerStringFunctions(const std::string& prefix) { registerFunction( {prefix + "to_title", prefix + "initcap"}); - BOLT_REGISTER_VECTOR_FUNCTION(udf_decimal_format_number, prefix + "format_number"); + BOLT_REGISTER_VECTOR_FUNCTION( + udf_decimal_format_number, prefix + "format_number"); registerFunction( {prefix + "format_number"}); registerFunction( diff --git a/bolt/functions/sparksql/tests/FormatNumberTest.cpp b/bolt/functions/sparksql/tests/FormatNumberTest.cpp index 9854a8a06..8b1fee3a5 100644 --- a/bolt/functions/sparksql/tests/FormatNumberTest.cpp +++ b/bolt/functions/sparksql/tests/FormatNumberTest.cpp @@ -53,8 +53,9 @@ class FormatNumberTest : public SparkFunctionBaseTest { int32_t decimalPlaces) { return evaluateOnce( "format_number(c0, c1)", - makeRowVector({makeFlatVector({unscaledValue}, type), - makeFlatVector({decimalPlaces})})); + makeRowVector( + {makeFlatVector({unscaledValue}, type), + makeFlatVector({decimalPlaces})})); } std::optional formatLongDecimal( @@ -63,8 +64,9 @@ class FormatNumberTest : public SparkFunctionBaseTest { int32_t decimalPlaces) { return evaluateOnce( "format_number(c0, c1)", - makeRowVector({makeFlatVector({unscaledValue}, type), - makeFlatVector({decimalPlaces})})); + makeRowVector( + {makeFlatVector({unscaledValue}, type), + makeFlatVector({decimalPlaces})})); } };