Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More efficient encoding of BigInt and java.math.BigInteger values #1323

Merged
merged 1 commit into from
Feb 18, 2025
+211 −60
Merged

More efficient encoding of BigInt and java.math.BigInteger values #1323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 100 additions & 30 deletions zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,20 @@ object SafeNumbers {
try Some(UnsafeNumbers.bigDecimal(num, max_bits))
catch { case _: UnexpectedEnd | UnsafeNumber => None }

def toString(x: java.math.BigInteger): String = {
val out = writes.get
write(x, out)
out.buffer.toString
}

def toString(x: Double): String = {
val out = new FastStringWrite(24)
val out = writes.get
write(x, out)
out.buffer.toString
}

def toString(x: Float): String = {
val out = new FastStringWrite(16)
val out = writes.get
write(x, out)
out.buffer.toString
}
Expand All @@ -96,6 +102,59 @@ object SafeNumbers {
out.buffer.toString
}

def write(x: java.math.BigInteger, out: Write): Unit = writeBigInteger(x, null, out)

private[this] def writeBigInteger(x: java.math.BigInteger, ss: Array[java.math.BigInteger], out: Write): Unit = {
val bitLen = x.bitLength
if (bitLen < 64) write(x.longValue, out)
else {
val n = calculateTenPow18SquareNumber(bitLen)
val ss1 =
if (ss eq null) getTenPow18Squares(n)
else ss
val qr = x.divideAndRemainder(ss1(n))
writeBigInteger(qr(0), ss1, out)
writeBigIntegerRemainder(qr(1), n - 1, ss1, out)
}
}

private[this] def writeBigIntegerRemainder(
x: java.math.BigInteger,
n: Int,
ss: Array[java.math.BigInteger],
out: Write
): Unit =
if (n < 0) write18Digits(Math.abs(x.longValue), out)
else {
val qr = x.divideAndRemainder(ss(n))
writeBigIntegerRemainder(qr(0), n - 1, ss, out)
writeBigIntegerRemainder(qr(1), n - 1, ss, out)
}

private[this] def calculateTenPow18SquareNumber(bitLen: Int): Int = {
val m = Math.max(
(bitLen * 0.016723888647998956).toInt - 1,
1
) // Math.max((x.bitLength * Math.log(2) / Math.log(1e18)).toInt - 1, 1)
31 - java.lang.Integer.numberOfLeadingZeros(m)
}

private[this] def getTenPow18Squares(n: Int): Array[java.math.BigInteger] = {
var ss = tenPow18Squares
var i = ss.length
if (n >= i) {
var s = ss(i - 1)
ss = java.util.Arrays.copyOf(ss, n + 1)
while (i <= n) {
s = s.multiply(s)
ss(i) = s
i += 1
}
tenPow18Squares = ss
}
ss
}

// Based on the amazing work of Raffaello Giulietti
// "The Schubfach way to render doubles": https://drive.google.com/file/d/1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN/view
// Sources with the license are here: https://github.com/c4f7fcce9cb06515/Schubfach/blob/3c92d3c9b1fead540616c918cdfef432bca53dfa/todec/src/math/DoubleToDecimal.java
Expand Down Expand Up @@ -343,16 +402,6 @@ object SafeNumbers {
write(stripTrailingZeros(x), out)
}

private[this] val writes = new ThreadLocal[FastStringWrite] {
override def initialValue(): FastStringWrite = new FastStringWrite(24)

override def get: FastStringWrite = {
val w = super.get
w.reset()
w
}
}

@inline private[this] def rop(g1: Long, g0: Long, cp: Long): Long = {
val x = multiplyHigh(g0, cp) + (g1 * cp >>> 1)
var y = multiplyHigh(g1, cp)
Expand Down Expand Up @@ -578,6 +627,14 @@ object SafeNumbers {
}
}

@inline private[this] def write18Digits(x: Long, out: Write): Unit = {
val q1 = ((x >>> 8) * 2.56e-6).toLong // divide a medium positive long by 100000000
val q2 = (q1 >>> 8) * 1441151881L >>> 49 // divide a small positive long by 100000000
out.write(digits(q2.toInt))
write8Digits((q1 - q2 * 100000000L).toInt, out)
write8Digits((x - q1 * 100000000L).toInt, out)
}

private[this] def write8Digits(x: Int, out: Write): Unit = {
val ds = digits
val q1 = x / 10000
Expand All @@ -603,24 +660,6 @@ object SafeNumbers {
@inline private[json] def write2Digits(x: Int, out: Write): Unit =
out.write(digits(x))

private[this] final val pow10ints: Array[Int] =
Array(1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000)

private[this] final val pow10longs: Array[Long] =
Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L,
100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L,
100000000000000000L)

private[this] final val digits: Array[Short] = Array(
12336, 12592, 12848, 13104, 13360, 13616, 13872, 14128, 14384, 14640, 12337, 12593, 12849, 13105, 13361, 13617,
13873, 14129, 14385, 14641, 12338, 12594, 12850, 13106, 13362, 13618, 13874, 14130, 14386, 14642, 12339, 12595,
12851, 13107, 13363, 13619, 13875, 14131, 14387, 14643, 12340, 12596, 12852, 13108, 13364, 13620, 13876, 14132,
14388, 14644, 12341, 12597, 12853, 13109, 13365, 13621, 13877, 14133, 14389, 14645, 12342, 12598, 12854, 13110,
13366, 13622, 13878, 14134, 14390, 14646, 12343, 12599, 12855, 13111, 13367, 13623, 13879, 14135, 14391, 14647,
12344, 12600, 12856, 13112, 13368, 13624, 13880, 14136, 14392, 14648, 12345, 12601, 12857, 13113, 13369, 13625,
13881, 14137, 14393, 14649
)

@inline
private[this] def digitCount(x: Long): Int =
if (x >= 1000000000000000L) {
Expand Down Expand Up @@ -655,6 +694,16 @@ object SafeNumbers {
else 10
}

private[this] final val digits: Array[Short] = Array(
12336, 12592, 12848, 13104, 13360, 13616, 13872, 14128, 14384, 14640, 12337, 12593, 12849, 13105, 13361, 13617,
13873, 14129, 14385, 14641, 12338, 12594, 12850, 13106, 13362, 13618, 13874, 14130, 14386, 14642, 12339, 12595,
12851, 13107, 13363, 13619, 13875, 14131, 14387, 14643, 12340, 12596, 12852, 13108, 13364, 13620, 13876, 14132,
14388, 14644, 12341, 12597, 12853, 13109, 13365, 13621, 13877, 14133, 14389, 14645, 12342, 12598, 12854, 13110,
13366, 13622, 13878, 14134, 14390, 14646, 12343, 12599, 12855, 13111, 13367, 13623, 13879, 14135, 14391, 14647,
12344, 12600, 12856, 13112, 13368, 13624, 13880, 14136, 14392, 14648, 12345, 12601, 12857, 13113, 13369, 13625,
13881, 14137, 14393, 14649
)

private[this] final val lowerCaseHexDigits: Array[Short] = Array(
12336, 12592, 12848, 13104, 13360, 13616, 13872, 14128, 14384, 14640, 24880, 25136, 25392, 25648, 25904, 26160,
12337, 12593, 12849, 13105, 13361, 13617, 13873, 14129, 14385, 14641, 24881, 25137, 25393, 25649, 25905, 26161,
Expand Down Expand Up @@ -919,4 +968,25 @@ object SafeNumbers {
8988465674311579538L, 5963149404718312264L, 7190772539449263630L, 8459868338516560134L, 5752618031559410904L,
6767894670813248108L, 9204188850495057447L, 5294608251188331487L
)

private[this] final val pow10ints: Array[Int] =
Array(1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000)

private[this] final val pow10longs: Array[Long] =
Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L,
100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L,
100000000000000000L)

@volatile private[this] var tenPow18Squares: Array[java.math.BigInteger] =
Array(java.math.BigInteger.valueOf(1000000000000000000L))

private[this] val writes = new ThreadLocal[FastStringWrite] {
override def initialValue(): FastStringWrite = new FastStringWrite(64)

override def get: FastStringWrite = {
val w = super.get
w.reset()
w
}
}
}
127 changes: 99 additions & 28 deletions zio-json/jvm-native/src/main/scala/zio/json/internal/SafeNumbers.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,20 @@ object SafeNumbers {
try Some(UnsafeNumbers.bigDecimal(num, max_bits))
catch { case _: UnexpectedEnd | UnsafeNumber => None }

def toString(x: java.math.BigInteger): String = {
val out = writes.get
write(x, out)
out.buffer.toString
}

def toString(x: Double): String = {
val out = new FastStringWrite(24)
val out = writes.get
write(x, out)
out.buffer.toString
}

def toString(x: Float): String = {
val out = new FastStringWrite(16)
val out = writes.get
write(x, out)
out.buffer.toString
}
Expand All @@ -96,6 +102,59 @@ object SafeNumbers {
out.buffer.toString
}

def write(x: java.math.BigInteger, out: Write): Unit = writeBigInteger(x, null, out)

private[this] def writeBigInteger(x: java.math.BigInteger, ss: Array[java.math.BigInteger], out: Write): Unit = {
val bitLen = x.bitLength
if (bitLen < 64) write(x.longValue, out)
else {
val n = calculateTenPow18SquareNumber(bitLen)
val ss1 =
if (ss eq null) getTenPow18Squares(n)
else ss
val qr = x.divideAndRemainder(ss1(n))
writeBigInteger(qr(0), ss1, out)
writeBigIntegerRemainder(qr(1), n - 1, ss1, out)
}
}

private[this] def writeBigIntegerRemainder(
x: java.math.BigInteger,
n: Int,
ss: Array[java.math.BigInteger],
out: Write
): Unit =
if (n < 0) write18Digits(Math.abs(x.longValue), out)
else {
val qr = x.divideAndRemainder(ss(n))
writeBigIntegerRemainder(qr(0), n - 1, ss, out)
writeBigIntegerRemainder(qr(1), n - 1, ss, out)
}

private[this] def calculateTenPow18SquareNumber(bitLen: Int): Int = {
val m = Math.max(
(bitLen * 71828554L >> 32).toInt - 1,
1
) // Math.max((x.bitLength * Math.log(2) / Math.log(1e18)).toInt - 1, 1)
31 - java.lang.Integer.numberOfLeadingZeros(m)
}

private[this] def getTenPow18Squares(n: Int): Array[java.math.BigInteger] = {
var ss = tenPow18Squares
var i = ss.length
if (n >= i) {
var s = ss(i - 1)
ss = java.util.Arrays.copyOf(ss, n + 1)
while (i <= n) {
s = s.multiply(s)
ss(i) = s
i += 1
}
tenPow18Squares = ss
}
ss
}

// Based on the amazing work of Raffaello Giulietti
// "The Schubfach way to render doubles": https://drive.google.com/file/d/1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN/view
// Sources with the license are here: https://github.com/c4f7fcce9cb06515/Schubfach/blob/3c92d3c9b1fead540616c918cdfef432bca53dfa/todec/src/math/DoubleToDecimal.java
Expand Down Expand Up @@ -334,16 +393,6 @@ object SafeNumbers {
write(stripTrailingZeros(x), out)
}

private[this] val writes = new ThreadLocal[FastStringWrite] {
override def initialValue(): FastStringWrite = new FastStringWrite(24)

override def get: FastStringWrite = {
val w = super.get
w.reset()
w
}
}

private[this] def rop(g1: Long, g0: Long, cp: Long): Long = {
val x = Math.multiplyHigh(g0, cp) + (g1 * cp >>> 1)
Math.multiplyHigh(g1, cp) + (x >>> 63) | (-x ^ x) >>> 63
Expand Down Expand Up @@ -400,9 +449,9 @@ object SafeNumbers {
else {
val q2 = Math.multiplyHigh(q1, m2) >>> 25 // divide a small positive long by 100000000
writeMantissa(q2.toInt, out)
write8Digits((q1 - q2 * m1).toInt, out)
write8Digits(q1 - q2 * m1, out)
}
write8Digits((q0 - q1 * m1).toInt, out)
write8Digits(q0 - q1 * m1, out)
}
}

Expand All @@ -411,15 +460,15 @@ object SafeNumbers {
else {
val q1 = Math.multiplyHigh(q0, 6189700196426901375L) >>> 25 // divide a positive long by 100000000
writeMantissa(q1.toInt, out)
write8Digits((q0 - q1 * 100000000L).toInt, out)
write8Digits(q0 - q1 * 100000000L, out)
}

private[this] def writeMantissaWithDot(q0: Long, out: Write): Unit =
if (q0.toInt == q0) writeMantissaWithDot(q0.toInt, out)
else {
val q1 = Math.multiplyHigh(q0, 6189700196426901375L) >>> 25 // divide a positive long by 100000000
writeMantissaWithDot(q1.toInt, out)
write8Digits((q0 - q1 * 100000000L).toInt, out)
write8Digits(q0 - q1 * 100000000L, out)
}

def write(a: Int, out: Write): Unit = {
Expand Down Expand Up @@ -526,7 +575,16 @@ object SafeNumbers {
}
}

private[this] def write8Digits(x: Int, out: Write): Unit = {
private[this] def write18Digits(x: Long, out: Write): Unit = {
val m1 = 6189700196426901375L
val q1 = Math.multiplyHigh(x, m1) >>> 25 // divide a positive long by 100000000
val q2 = Math.multiplyHigh(q1, m1) >>> 25 // divide a positive long by 100000000
out.write(digits(q2.toInt))
write8Digits(q1 - q2 * 100000000L, out)
write8Digits(x - q1 * 100000000L, out)
}

private[this] def write8Digits(x: Long, out: Write): Unit = {
val ds = digits // Based on James Anhalt's algorithm: https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/
val y1 = x * 140737489L
val m1 = 0x7fffffffffffL
Expand All @@ -552,13 +610,9 @@ object SafeNumbers {
@inline private[json] def write2Digits(x: Int, out: Write): Unit =
out.write(digits(x))

private[this] final val pow10ints: Array[Int] =
Array(1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000)

private[this] final val pow10longs: Array[Long] =
Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L,
100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L,
100000000000000000L)
// Adoption of a nice trick form Daniel Lemire's blog that works for numbers up to 10^18:
// https://lemire.me/blog/2021/06/03/computing-the-number-of-digits-of-an-integer-even-faster/
private[this] def digitCount(x: Long): Int = (offsets(java.lang.Long.numberOfLeadingZeros(x)) + x >> 58).toInt

private[this] final val digits: Array[Short] = Array(
12336, 12592, 12848, 13104, 13360, 13616, 13872, 14128, 14384, 14640, 12337, 12593, 12849, 13105, 13361, 13617,
Expand All @@ -570,10 +624,6 @@ object SafeNumbers {
13881, 14137, 14393, 14649
)

// Adoption of a nice trick form Daniel Lemire's blog that works for numbers up to 10^18:
// https://lemire.me/blog/2021/06/03/computing-the-number-of-digits-of-an-integer-even-faster/
private[this] def digitCount(x: Long): Int = (offsets(java.lang.Long.numberOfLeadingZeros(x)) + x >> 58).toInt

private[this] val offsets = Array(
5088146770730811392L, 5088146770730811392L, 5088146770730811392L, 5088146770730811392L, 5088146770730811392L,
5088146770730811392L, 5088146770730811392L, 5088146770730811392L, 4889916394579099648L, 4889916394579099648L,
Expand Down Expand Up @@ -854,4 +904,25 @@ object SafeNumbers {
8988465674311579538L, 5963149404718312264L, 7190772539449263630L, 8459868338516560134L, 5752618031559410904L,
6767894670813248108L, 9204188850495057447L, 5294608251188331487L
)

private[this] final val pow10ints: Array[Int] =
Array(1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000)

private[this] final val pow10longs: Array[Long] =
Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L,
100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L,
100000000000000000L)

@volatile private[this] var tenPow18Squares: Array[java.math.BigInteger] =
Array(java.math.BigInteger.valueOf(1000000000000000000L))

private[this] val writes = new ThreadLocal[FastStringWrite] {
override def initialValue(): FastStringWrite = new FastStringWrite(64)

override def get: FastStringWrite = {
val w = super.get
w.reset()
w
}
}
}
Loading
Loading