From 668673a240e7d564ca709d4fd32bb2b9b575abea Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Fri, 7 Jan 2022 03:58:57 -0500 Subject: [PATCH 1/5] apd: split round and goError calls to permit inlining `Context.Round` includes a call to `Context.round` and `Context.goError`. Of these three, only `Context.goError` can be inlined. To avoid a function call on performance sensitive code paths, we split a few calls to Context.Round into separate calls to `Context.round` and `Context.goError`. We were already doing this in some places. --- context.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/context.go b/context.go index 4b3a96c..3a40124 100644 --- a/context.go +++ b/context.go @@ -79,6 +79,7 @@ func (c *Context) WithPrecision(p uint32) *Context { } // goError converts flags into an error based on c.Traps. +//gcassert:inline func (c *Context) goError(flags Condition) (Condition, error) { return flags.GoError(c.Traps) } @@ -156,7 +157,8 @@ func (c *Context) add(d, x, y *Decimal, subtract bool) (Condition, error) { } d.Exponent = s d.Form = Finite - return c.Round(d, d) + res := c.round(d, d) + return c.goError(res) } // Add sets d to the sum x+y. @@ -175,7 +177,8 @@ func (c *Context) Abs(d, x *Decimal) (Condition, error) { return res, err } d.Abs(x) - return c.Round(d, d) + res := c.round(d, d) + return c.goError(res) } // Neg sets d to -x. @@ -184,7 +187,8 @@ func (c *Context) Neg(d, x *Decimal) (Condition, error) { return res, err } d.Neg(x) - return c.Round(d, d) + res := c.round(d, d) + return c.goError(res) } // Mul sets d to the product x*y. @@ -552,7 +556,8 @@ func (c *Context) Sqrt(d, x *Decimal) (Condition, error) { nc.Precision = c.Precision nc.Rounding = RoundHalfEven d.Reduce(d) // Remove trailing zeros. - return nc.Round(d, d) + res := nc.round(d, d) + return nc.goError(res) } // Cbrt sets d to the cube root of x. @@ -631,7 +636,8 @@ func (c *Context) Cbrt(d, x *Decimal) (Condition, error) { } z0.Set(x) - res, err := c.Round(d, &z) + res := c.round(d, &z) + res, err := c.goError(res) d.Negative = neg // Set z = d^3 to check for exactness. @@ -1278,7 +1284,8 @@ func (c *Context) Reduce(d, x *Decimal) (int, Condition, error) { neg := x.Negative _, n := d.Reduce(x) d.Negative = neg - res, err := c.Round(d, d) + res := c.round(d, d) + res, err := c.goError(res) return n, res, err } From cce190ee8e442d263e9bd6a72da0b0fa8cb08869 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Fri, 7 Jan 2022 04:05:12 -0500 Subject: [PATCH 2/5] apd: inline aliasing fast-path of Decimal.Set Split out a "slow" path so that the header can be inlined in callers. --- decimal.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/decimal.go b/decimal.go index 443ccc6..9cd8f9f 100644 --- a/decimal.go +++ b/decimal.go @@ -197,14 +197,21 @@ func (c *Context) SetString(d *Decimal, s string) (*Decimal, Condition, error) { } // Set sets d's fields to the values of x and returns d. +//gcassert:inline func (d *Decimal) Set(x *Decimal) *Decimal { if d == x { return d } + return d.setSlow(x) +} + +// setSlow is split from Set to allow the aliasing fast-path to be +// inlined in callers. +func (d *Decimal) setSlow(x *Decimal) *Decimal { + d.Form = x.Form d.Negative = x.Negative - d.Coeff.Set(&x.Coeff) d.Exponent = x.Exponent - d.Form = x.Form + d.Coeff.Set(&x.Coeff) return d } From 5f543ef978537cf489076016e01f82d7e25270d7 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Fri, 7 Jan 2022 04:12:11 -0500 Subject: [PATCH 3/5] apd: avoid redundant calls to NumDigits These calls are expensive, so avoid redundant calls in `Rounder.Round`. --- context.go | 4 ++-- decimal.go | 18 ++++++++++++------ round.go | 8 +++++--- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/context.go b/context.go index 3a40124..043bfde 100644 --- a/context.go +++ b/context.go @@ -211,7 +211,7 @@ func (c *Context) Mul(d, x, y *Decimal) (Condition, error) { d.Coeff.Mul(&x.Coeff, &y.Coeff) d.Negative = neg d.Form = Finite - res := d.setExponent(c, 0, int64(x.Exponent), int64(y.Exponent)) + res := d.setExponent(c, unknownNumDigits, 0, int64(x.Exponent), int64(y.Exponent)) res |= c.round(d, d) return c.goError(res) } @@ -364,7 +364,7 @@ func (c *Context) Quo(d, x, y *Decimal) (Condition, error) { // The exponent of the result is computed by subtracting the sum of the // original exponent of the divisor and the value of adjust at the end of // the coefficient calculation from the original exponent of the dividend. - res |= quo.setExponent(c, res, int64(x.Exponent), int64(-y.Exponent), -adjust, diff) + res |= quo.setExponent(c, unknownNumDigits, res, int64(x.Exponent), int64(-y.Exponent), -adjust, diff) quo.Negative = neg d.Set(&quo) return c.goError(res) diff --git a/decimal.go b/decimal.go index 9cd8f9f..1e37dd3 100644 --- a/decimal.go +++ b/decimal.go @@ -160,7 +160,7 @@ func (d *Decimal) setString(c *Context, s string) (Condition, error) { } // No parse errors, can now flag as finite. d.Form = Finite - return c.goError(d.setExponent(c, 0, exps...)) + return c.goError(d.setExponent(c, unknownNumDigits, 0, exps...)) } // NewFromString creates a new decimal from s. It has no restrictions on @@ -283,14 +283,18 @@ func (d *Decimal) Float64() (float64, error) { const ( errExponentOutOfRangeStr = "exponent out of range" + + unknownNumDigits = int64(-1) ) // setExponent sets d's Exponent to the sum of xs. Each value and the sum // of xs must fit within an int32. An error occurs if the sum is outside of -// the MaxExponent or MinExponent range. res is any Condition previously set -// for this operation, which can cause Underflow to be set if, for example, -// Inexact is already set. -func (d *Decimal) setExponent(c *Context, res Condition, xs ...int64) Condition { +// the MaxExponent or MinExponent range. nd is the number of digits in d, as +// computed by NumDigits. Callers can pass unknownNumDigits to indicate that +// they have not yet computed this digit count, in which case setExponent will +// do so. res is any Condition previously set for this operation, which can +// cause Underflow to be set if, for example, Inexact is already set. +func (d *Decimal) setExponent(c *Context, nd int64, res Condition, xs ...int64) Condition { var sum int64 for _, x := range xs { if x > MaxExponent { @@ -303,7 +307,9 @@ func (d *Decimal) setExponent(c *Context, res Condition, xs ...int64) Condition } r := int32(sum) - nd := d.NumDigits() + if nd == unknownNumDigits { + nd = d.NumDigits() + } // adj is the adjusted exponent: exponent + clength - 1 adj := sum + nd - 1 // Make sure it is less than the system limits. diff --git a/round.go b/round.go index 4cea210..92998c5 100644 --- a/round.go +++ b/round.go @@ -24,7 +24,7 @@ func (c *Context) Round(d, x *Decimal) (Condition, error) { func (c *Context) round(d, x *Decimal) Condition { if c.Precision == 0 { d.Set(x) - return d.setExponent(c, 0, int64(d.Exponent)) + return d.setExponent(c, unknownNumDigits, 0, int64(d.Exponent)) } res := c.Rounding.Round(c, d, x) return res @@ -74,7 +74,7 @@ func (r Rounder) Round(c *Context, d, x *Decimal) Condition { // Subnormal is defined before rounding. res |= Subnormal // setExponent here to prevent double-rounded subnormals. - res |= d.setExponent(c, res, int64(d.Exponent)) + res |= d.setExponent(c, nd, res, int64(d.Exponent)) return res } @@ -100,10 +100,12 @@ func (r Rounder) Round(c *Context, d, x *Decimal) Condition { } } d.Coeff.Set(&y) + // The coefficient changed, so recompute num digits in setExponent. + nd = unknownNumDigits } else { diff = 0 } - res |= d.setExponent(c, res, int64(d.Exponent), diff) + res |= d.setExponent(c, nd, res, int64(d.Exponent), diff) return res } From 502d7a2a70d6231f589fb1266f293566b75e72d0 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Fri, 7 Jan 2022 03:02:51 -0500 Subject: [PATCH 4/5] apd: manually unroll loops in BigInt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` name old time/op new time/op delta GDA/powersqrt-10 243ms ± 0% 197ms ± 0% -18.89% (p=0.000 n=10+10) GDA/squareroot-10 15.7ms ± 0% 12.8ms ± 0% -18.71% (p=0.000 n=10+10) GDA/rounding-10 302µs ± 1% 264µs ± 1% -12.41% (p=0.000 n=10+10) GDA/divide-10 267µs ± 1% 237µs ± 0% -11.14% (p=0.000 n=10+10) GDA/randoms-10 1.89ms ± 0% 1.77ms ± 0% -6.13% (p=0.000 n=10+10) GDA/cuberoot-apd-10 1.93ms ± 0% 1.82ms ± 0% -6.02% (p=0.000 n=9+9) GDA/ln-10 84.9ms ± 1% 80.7ms ± 0% -4.94% (p=0.000 n=10+9) GDA/log10-10 109ms ± 1% 104ms ± 0% -4.55% (p=0.000 n=10+10) GDA/power-10 226ms ± 0% 217ms ± 1% -4.18% (p=0.000 n=10+10) GDA/divideint-10 15.8µs ± 1% 15.2µs ± 1% -3.52% (p=0.000 n=10+9) GDA/exp-10 125ms ± 1% 121ms ± 1% -3.30% (p=0.000 n=10+10) GDA/remainder-10 30.7µs ± 1% 29.7µs ± 1% -3.02% (p=0.000 n=10+10) GDA/comparetotal-10 25.4µs ± 1% 24.7µs ± 1% -2.71% (p=0.000 n=10+10) GDA/quantize-10 76.1µs ± 0% 74.3µs ± 1% -2.38% (p=0.000 n=8+10) GDA/subtract-10 90.6µs ± 0% 88.8µs ± 1% -2.02% (p=0.000 n=9+10) GDA/tointegral-10 17.9µs ± 1% 17.5µs ± 1% -1.99% (p=0.000 n=10+10) GDA/reduce-10 12.0µs ± 1% 11.8µs ± 1% -1.77% (p=0.000 n=10+10) GDA/tointegralx-10 18.2µs ± 1% 17.9µs ± 1% -1.76% (p=0.000 n=10+10) GDA/multiply-10 54.1µs ± 0% 53.3µs ± 1% -1.44% (p=0.000 n=9+10) GDA/compare-10 28.8µs ± 1% 28.4µs ± 1% -1.41% (p=0.000 n=10+10) GDA/minus-10 6.86µs ± 1% 6.79µs ± 0% -0.99% (p=0.000 n=10+9) GDA/plus-10 35.5µs ± 1% 35.2µs ± 1% -0.98% (p=0.000 n=10+10) GDA/add-10 586µs ± 1% 581µs ± 0% -0.86% (p=0.000 n=10+9) GDA/abs-10 5.19µs ± 1% 5.16µs ± 1% ~ (p=0.171 n=10+10) GDA/base-10 116µs ± 0% 116µs ± 0% ~ (p=0.684 n=10+10) ``` --- bigint.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/bigint.go b/bigint.go index ab89ebc..c2b6e64 100644 --- a/bigint.go +++ b/bigint.go @@ -214,11 +214,20 @@ func (z *BigInt) innerAsUint() (val uint, neg bool, ok bool) { // The value is not stored inline. return 0, false, false } - for i := 1; i < len(z._inline); i++ { - if z._inline[i] != 0 { + if inlineWords == 2 { + // Manually unrolled loop for current inlineWords setting. + if z._inline[1] != 0 { // The value can not fit in a uint. return 0, false, false } + } else { + // Fallback for other values of inlineWords. + for i := 1; i < len(z._inline); i++ { + if z._inline[i] != 0 { + // The value can not fit in a uint. + return 0, false, false + } + } } val = uint(z._inline[0]) @@ -235,8 +244,14 @@ func (z *BigInt) innerAsUint() (val uint, neg bool, ok bool) { func (z *BigInt) updateInnerFromUint(val uint, neg bool) { // Set the inline value, making sure to clear out all other words. z._inline[0] = big.Word(val) - for i := 1; i < len(z._inline); i++ { - z._inline[i] = 0 + if inlineWords == 2 { + // Manually unrolled loop for current inlineWords setting. + z._inline[1] = 0 + } else { + // Fallback for other values of inlineWords. + for i := 1; i < len(z._inline); i++ { + z._inline[i] = 0 + } } // Set or unset the negative sentinel. From e6e6903aabd4096839a91a3135575d777d16f4b5 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Fri, 7 Jan 2022 03:52:57 -0500 Subject: [PATCH 5/5] apd: inline "should set as NaN" decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids a function call in the prelude of each operation. ``` name old time/op new time/op delta GDA/abs-10 5.15µs ± 1% 4.83µs ± 0% -6.22% (p=0.000 n=10+9) GDA/subtract-10 88.8µs ± 0% 86.2µs ± 0% -2.91% (p=0.000 n=10+8) GDA/multiply-10 53.5µs ± 1% 52.0µs ± 0% -2.91% (p=0.000 n=10+10) GDA/add-10 580µs ± 1% 572µs ± 0% -1.24% (p=0.000 n=10+9) ``` --- context.go | 98 ++++++++++++++++++++++++++++++------------------------ decimal.go | 4 +-- 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/context.go b/context.go index 043bfde..e84d330 100644 --- a/context.go +++ b/context.go @@ -89,25 +89,31 @@ func (c *Context) etiny() int32 { return c.MinExponent - int32(c.Precision) + 1 } -// setIfNaN sets d to the first NaNSignaling, or otherwise first NaN, of -// vals. d' is unchanged if vals contains no NaNs. True is returned if d -// was set to a NaN. -func (c *Context) setIfNaN(d *Decimal, vals ...*Decimal) (bool, Condition, error) { +// shouldSetAsNaN determines whether setAsNaN should be called, given +// the provided values, where x is required and y is optional. It is +// split from setAsNaN to permit inlining of this function. +//gcassert:inline +func (c *Context) shouldSetAsNaN(x, y *Decimal) bool { + return x.Form == NaNSignaling || x.Form == NaN || + (y != nil && (y.Form == NaNSignaling || y.Form == NaN)) +} + +// setAsNaN sets d to the first NaNSignaling, or otherwise first NaN, of +// x and y. x is required, y is optional. Expects one of the two inputs +// to be NaN. +func (c *Context) setAsNaN(d *Decimal, x, y *Decimal) (Condition, error) { var nan *Decimal -Loop: - for _, v := range vals { - switch v.Form { - case NaNSignaling: - nan = v - break Loop - case NaN: - if nan == nil { - nan = v - } - } - } - if nan == nil { - return false, 0, nil + // Per the method contract, NaNSignaling takes precedence over NaN. + if x.Form == NaNSignaling { + nan = x + } else if y != nil && y.Form == NaNSignaling { + nan = y + } else if x.Form == NaN { + nan = x + } else if y != nil && y.Form == NaN { + nan = y + } else { + return 0, errors.Errorf("no NaN value found; was shouldSetAsNaN called?") } d.Set(nan) var res Condition @@ -116,12 +122,12 @@ Loop: d.Form = NaN } _, err := c.goError(res) - return true, res, err + return res, err } func (c *Context) add(d, x, y *Decimal, subtract bool) (Condition, error) { - if set, res, err := c.setIfNaN(d, x, y); set { - return res, err + if c.shouldSetAsNaN(x, y) { + return c.setAsNaN(d, x, y) } xn := x.Negative yn := y.Negative != subtract @@ -173,8 +179,8 @@ func (c *Context) Sub(d, x, y *Decimal) (Condition, error) { // Abs sets d to |x| (the absolute value of x). func (c *Context) Abs(d, x *Decimal) (Condition, error) { - if set, res, err := c.setIfNaN(d, x); set { - return res, err + if c.shouldSetAsNaN(x, nil) { + return c.setAsNaN(d, x, nil) } d.Abs(x) res := c.round(d, d) @@ -183,8 +189,8 @@ func (c *Context) Abs(d, x *Decimal) (Condition, error) { // Neg sets d to -x. func (c *Context) Neg(d, x *Decimal) (Condition, error) { - if set, res, err := c.setIfNaN(d, x); set { - return res, err + if c.shouldSetAsNaN(x, nil) { + return c.setAsNaN(d, x, nil) } d.Neg(x) res := c.round(d, d) @@ -193,8 +199,8 @@ func (c *Context) Neg(d, x *Decimal) (Condition, error) { // Mul sets d to the product x*y. func (c *Context) Mul(d, x, y *Decimal) (Condition, error) { - if set, res, err := c.setIfNaN(d, x, y); set { - return res, err + if c.shouldSetAsNaN(x, y) { + return c.setAsNaN(d, x, y) } // The sign of the result is the exclusive or of the signs of the operands. neg := x.Negative != y.Negative @@ -217,9 +223,11 @@ func (c *Context) Mul(d, x, y *Decimal) (Condition, error) { } func (c *Context) quoSpecials(d, x, y *Decimal, canClamp bool) (bool, Condition, error) { - if set, res, err := c.setIfNaN(d, x, y); set { + if c.shouldSetAsNaN(x, y) { + res, err := c.setAsNaN(d, x, y) return true, res, err } + // The sign of the result is the exclusive or of the signs of the operands. neg := x.Negative != y.Negative if xi, yi := x.Form == Infinite, y.Form == Infinite; xi || yi { @@ -400,8 +408,8 @@ func (c *Context) QuoInteger(d, x, y *Decimal) (Condition, error) { // Rem sets d to the remainder part of the quotient x/y. If // the integer part cannot fit in d.Precision digits, an error is returned. func (c *Context) Rem(d, x, y *Decimal) (Condition, error) { - if set, res, err := c.setIfNaN(d, x, y); set { - return res, err + if c.shouldSetAsNaN(x, y) { + return c.setAsNaN(d, x, y) } if x.Form != Finite { @@ -443,8 +451,9 @@ func (c *Context) Rem(d, x, y *Decimal) (Condition, error) { } func (c *Context) rootSpecials(d, x *Decimal, factor int32) (bool, Condition, error) { - if set, res, err := c.setIfNaN(d, x); set { - return set, res, err + if c.shouldSetAsNaN(x, nil) { + res, err := c.setAsNaN(d, x, nil) + return true, res, err } if x.Form == Infinite { if x.Negative { @@ -656,8 +665,9 @@ func (c *Context) Cbrt(d, x *Decimal) (Condition, error) { } func (c *Context) logSpecials(d, x *Decimal) (bool, Condition, error) { - if set, res, err := c.setIfNaN(d, x); set { - return set, res, err + if c.shouldSetAsNaN(x, nil) { + res, err := c.setAsNaN(d, x, nil) + return true, res, err } if x.Sign() < 0 { d.Set(decimalNaN) @@ -879,8 +889,8 @@ func (c *Context) Exp(d, x *Decimal) (Condition, error) { // See: Variable Precision Exponential Function, T. E. Hull and A. Abrham, ACM // Transactions on Mathematical Software, Vol 12 #2, pp79-91, ACM, June 1986. - if set, res, err := c.setIfNaN(d, x); set { - return res, err + if c.shouldSetAsNaN(x, nil) { + return c.setAsNaN(d, x, nil) } if x.Form == Infinite { if x.Negative { @@ -1043,8 +1053,8 @@ func (c *Context) integerPower(d, x *Decimal, y *BigInt) (Condition, error) { // Pow sets d = x**y. func (c *Context) Pow(d, x, y *Decimal) (Condition, error) { - if set, res, err := c.setIfNaN(d, x, y); set { - return res, err + if c.shouldSetAsNaN(x, y) { + return c.setAsNaN(d, x, y) } var integ, frac Decimal @@ -1151,8 +1161,8 @@ func (c *Context) Pow(d, x, y *Decimal) (Condition, error) { // Quantize adjusts and rounds x as necessary so it is represented with // exponent exp and stores the result in d. func (c *Context) Quantize(d, x *Decimal, exp int32) (Condition, error) { - if set, res, err := c.setIfNaN(d, x); set { - return res, err + if c.shouldSetAsNaN(x, nil) { + return c.setAsNaN(d, x, nil) } if x.Form == Infinite || exp < c.etiny() { d.Set(decimalNaN) @@ -1225,8 +1235,9 @@ func (c *Context) toIntegral(d, x *Decimal) Condition { } func (c *Context) toIntegralSpecials(d, x *Decimal) (bool, Condition, error) { - if set, res, err := c.setIfNaN(d, x); set { - return set, res, err + if c.shouldSetAsNaN(x, nil) { + res, err := c.setAsNaN(d, x, nil) + return true, res, err } if x.Form != Finite { d.Set(x) @@ -1278,7 +1289,8 @@ func (c *Context) Floor(d, x *Decimal) (Condition, error) { // Reduce sets d to x with all trailing zeros removed and returns the number // of zeros removed. func (c *Context) Reduce(d, x *Decimal) (int, Condition, error) { - if set, res, err := c.setIfNaN(d, x); set { + if c.shouldSetAsNaN(x, nil) { + res, err := c.setAsNaN(d, x, nil) return 0, res, err } neg := x.Negative diff --git a/decimal.go b/decimal.go index 1e37dd3..03c8a88 100644 --- a/decimal.go +++ b/decimal.go @@ -499,8 +499,8 @@ func (d *Decimal) cmpOrder() int { // This comparison respects the normal rules of special values (like NaN), // and does not compare them. func (c *Context) Cmp(d, x, y *Decimal) (Condition, error) { - if set, res, err := c.setIfNaN(d, x, y); set { - return res, err + if c.shouldSetAsNaN(x, y) { + return c.setAsNaN(d, x, y) } v := x.Cmp(y) d.SetInt64(int64(v))