From c669894213f3525ac4096c5f4e29c3876f5c08b7 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 14:02:23 +0800 Subject: [PATCH 01/17] add testSlow --- bench/src-js/Main.scala | 62 ++++-- bench/src-jvm/Main.scala | 187 +++++++++--------- bench/src-native/Main.scala | 44 ++++- bench/src/Common.scala | 96 ++++++++- bench/src/Micro.scala | 55 ++++++ build.sc | 14 +- .../src-js-native/upickle/withTimeout.scala | 9 + .../src-jvm/upickle/withTimeout.scala | 16 ++ upickle/testSlow/src/upickle/SlowTests.scala | 121 ++++++++++++ 9 files changed, 487 insertions(+), 117 deletions(-) create mode 100644 bench/src/Micro.scala create mode 100644 upickle/testSlow/src-js-native/upickle/withTimeout.scala create mode 100644 upickle/testSlow/src-jvm/upickle/withTimeout.scala create mode 100644 upickle/testSlow/src/upickle/SlowTests.scala diff --git a/bench/src-js/Main.scala b/bench/src-js/Main.scala index 437076142..0096c7599 100644 --- a/bench/src-js/Main.scala +++ b/bench/src-js/Main.scala @@ -8,33 +8,69 @@ import upickle.Recursive.{End, LL, Node} import scala.scalajs.js object Main{ def main(args: Array[String]): Unit = { - for(duration <- Seq(2500, 5000, 10000, 10000, 10000, 10000)){ + val allResults = collection.mutable.Buffer.empty[(String, Int)] + for((duration, save) <- Seq(2500 -> false, 5000 -> false, 10000 -> true, 10000 -> true, 10000 -> true)){ println("RUN JS: " + duration) println() // rawJsonParseSerialize(duration) - NonNative.playJson(duration) - NonNative.circe(duration) - Common.upickleDefault(duration) - Common.upickleDefaultByteArray(duration) - Common.upickleDefaultBinary(duration) - upickleWebDefault(duration) +// NonNative.playJson(duration) +// NonNative.circe(duration) +// Common.upickleDefault(duration) +// Common.upickleDefaultByteArray(duration) +// Common.upickleDefaultBinary(duration) +// upickleWebDefault(duration) // Common.upickleLegacy(duration) // Common.upickleBinaryLegacy(duration) // Common.genCodec(duration) // upickleWebDefault(duration) // upickleWebLegacy(duration) - NonNative.playJsonCached(duration) - NonNative.circeCached(duration) - upickleWebDefaultCached(duration) - Common.upickleDefaultCached(duration) - Common.upickleDefaultByteArrayCached(duration) - Common.upickleDefaultBinaryCached(duration) +// NonNative.playJsonCached(duration) +// NonNative.circeCached(duration) +// upickleWebDefaultCached(duration) +// Common.upickleDefaultCached(duration) +// Common.upickleDefaultByteArrayCached(duration) +// Common.upickleDefaultBinaryCached(duration) // Common.upickleLegacyCached(duration) // Common.upickleLegacyBinaryCached(duration) // Common.genCodecCached(duration) // upickleWebLegacyCached(duration) + val results = Seq( + Common.upickleDefault(duration), + Common.upickleDefaultByteArray(duration), + Common.upickleDefaultBinary(duration), + Common.integers(duration), + Common.integersByteArray(duration), + Common.integersBinary(duration), + + Common.doubles(duration), + Common.doublesByteArray(duration), + Common.doublesBinary(duration), + + Common.sequences(duration), + Common.sequencesByteArray(duration), + Common.sequencesBinary(duration), + + Common.shortStrings(duration), + Common.shortStringsByteArray(duration), + Common.shortStringsBinary(duration), + + Common.longStrings(duration), + Common.longStringsByteArray(duration), + Common.longStringsBinary(duration), + + Common.unicodeStrings(duration), + Common.unicodeStringsByteArray(duration), + Common.unicodeStringsBinary(duration), + + Common.caseClasses(duration), + Common.caseClassesByteArray(duration), + Common.caseClassesBinary(duration) + ) + println() + if (save) allResults.appendAll(results.flatten) println() } + Common.prettyPrintResults(allResults) } def rawJsonParseSerialize(duration: Int) = { diff --git a/bench/src-jvm/Main.scala b/bench/src-jvm/Main.scala index e9a4e3419..e704728df 100644 --- a/bench/src-jvm/Main.scala +++ b/bench/src-jvm/Main.scala @@ -11,49 +11,109 @@ object Main{ import Hierarchy._ import Recursive._ def main(args: Array[String]): Unit = { - for(duration <- Seq(2500, 5000, 10000, 10000, 10000, 10000)){ + val allResults = collection.mutable.Buffer.empty[(String, Int)] + for((duration, save) <- Seq(2500 -> false, 5000 -> false, 10000 -> true, 10000 -> true, 10000 -> true)){ println("RUN JVM: " + duration) println() -// Main.ujsonAst(duration) -// Main.upackAst(duration) -// Main.playJsonAst(duration) -// Main.uJsonPlayJsonAst(duration) -// Main.circeJsonAst(duration) -// Main.uJsonCirceJsonAst(duration) -// Main.argonautJsonAst(duration) -// Main.uJsonArgonautJsonAst(duration) -// Main.json4sJsonAst(duration) -// Main.uJsonJson4sJsonAst(duration) - -// Main.jacksonModuleScala(duration) - NonNative.playJson(duration) - NonNative.circe(duration) - Common.upickleDefault(duration) - Common.upickleDefaultByteArray(duration) - Common.upickleDefaultBinary(duration) - // Common.upickleLegacy(duration) -// Common.upickleBinaryLegacy(duration) -// Common.genCodec(duration) - NonNative.playJsonCached(duration) - NonNative.circeCached(duration) - Common.upickleDefaultCached(duration) - Common.upickleDefaultByteArrayCached(duration) - Common.upickleDefaultBinaryCached(duration) - // Common.upickleDefaultCachedReadable(duration) - // Common.upickleDefaultCachedReadablePath(duration) - - // Common.upickleLegacyCached(duration) -// Common.upickleDefaultBinaryCachedReadable(duration) -// Common.upickleLegacyBinaryCached(duration) -// Common.genCodecCached(duration) - benchParsingRendering(duration, bytes = true, strings = false, streams = false, msgpack = false) - benchParsingRendering(duration, bytes = false, strings = true, streams = false, msgpack = false) - benchParsingRendering(duration, bytes = false, strings = false, streams = true, msgpack = false) - benchParsingRendering(duration, bytes = false, strings = false, streams = false, msgpack = true) + // Main.ujsonAst(duration) + // Main.upackAst(duration) + // Main.playJsonAst(duration) + // Main.uJsonPlayJsonAst(duration) + // Main.circeJsonAst(duration) + // Main.uJsonCirceJsonAst(duration) + // Main.argonautJsonAst(duration) + // Main.uJsonArgonautJsonAst(duration) + // Main.json4sJsonAst(duration) + // Main.uJsonJson4sJsonAst(duration) + + // Main.jacksonModuleScala(duration) + + + val results = Seq( +// NonNative.playJson(duration), +// NonNative.circe(duration), + Common.upickleDefault(duration), + Common.upickleDefaultByteArray(duration), + Common.upickleDefaultBinary(duration), + // Common.upickleLegacy(duration) + // Common.upickleBinaryLegacy(duration) + // Common.genCodec(duration) + // NonNative.playJsonCached(duration) + // NonNative.circeCached(duration) + // Common.upickleDefaultCached(duration) + // Common.upickleDefaultByteArrayCached(duration) + // Common.upickleDefaultBinaryCached(duration) + // Common.upickleDefaultCachedReadable(duration) + // Common.upickleDefaultCachedReadablePath(duration) + + // Common.upickleLegacyCached(duration) + // Common.upickleDefaultBinaryCachedReadable(duration) + // Common.upickleLegacyBinaryCached(duration) + // Common.genCodecCached(duration) + // benchParsingRendering(duration, bytes = true, strings = false, streams = false, msgpack = false) + // benchParsingRendering(duration, bytes = false, strings = true, streams = false, msgpack = false) + // benchParsingRendering(duration, bytes = false, strings = false, streams = true, msgpack = false) + // benchParsingRendering(duration, bytes = false, strings = false, streams = false, msgpack = true) + Common.integers(duration), + Common.integersByteArray(duration), + Common.integersBinary(duration), + + Common.doubles(duration), + Common.doublesByteArray(duration), + Common.doublesBinary(duration), + + Common.shortStrings(duration), + Common.shortStringsByteArray(duration), + Common.shortStringsBinary(duration), + + Common.longStrings(duration), + Common.longStringsByteArray(duration), + Common.longStringsBinary(duration), + + Common.unicodeStrings(duration), + Common.unicodeStringsByteArray(duration), + Common.unicodeStringsBinary(duration), + + Common.caseClasses(duration), + Common.caseClassesByteArray(duration), + Common.caseClassesBinary(duration), + + Common.sequences(duration), + Common.sequencesByteArray(duration), + Common.sequencesBinary(duration), + ) + + if (save) allResults.appendAll(results.flatten) println() } + val prettyResults = Common.prettyPrintResults(allResults) + println(prettyResults) + for(savePath <- args.headOption){ + val p = java.nio.file.Paths.get(savePath) + if (!java.nio.file.Files.exists(p)) java.nio.file.Files.write(p, prettyResults.getBytes) + else { + val before = new String(java.nio.file.Files.readAllBytes(p)) + def split(s: String) = s.linesIterator.map(_.drop(2).dropRight(2).split(" *\\| *")).toList + println( + split(prettyResults).zip(split(before)).zipWithIndex.map { + case ((afterLine, beforeLine), 0 | 1) => afterLine + case ((afterLine, beforeLine), _) => + afterLine.zip(beforeLine) match { + case Array(head, tail@_*) => + Array(head._1) ++ tail.map { case (a, b) => + val percentChange = ((a.toDouble / b.toDouble - 1) * 100).toInt + val sign = + if (percentChange == 0) "" else if (percentChange < 0) "-" else if (percentChange > 0) "+" + s"$b -> $a (${sign}${math.abs(percentChange)}%)" + } + } + }.map(_.mkString("| ", " | ", " |")).mkString("\n") + ) + } + } } + @nowarn("cat=deprecation") def benchParsingRendering(duration: Int, bytes: Boolean, strings: Boolean, streams: Boolean, msgpack: Boolean) = { import java.nio.file.{Files, Paths} @@ -163,57 +223,4 @@ object Main{ ujson.json4s.Json4sJson.transform(_, ujson.StringRenderer()).toString ) } -// def jacksonModuleScala(duration: Int) = { -// val mapper = new ObjectMapper() with ScalaObjectMapper -// val m = new SimpleModule -// mapper.registerModule(DefaultScalaModule) -// -// // https://stackoverflow.com/questions/47955581/jackson-deserialize-json-to-scala-adt?rq=1 -// m.addDeserializer( -// classOf[A], -// new StdDeserializer[A](classOf[A]) { -// def deserialize(jp: JsonParser, ctxt: DeserializationContext): A = { -// val tb = new TokenBuffer(jp, ctxt) -// tb.copyCurrentStructure(jp) -// val firstParser = tb.asParser -// firstParser.nextToken -// val curNode = firstParser.getCodec.readTree[JsonNode](firstParser) -// val objectParser = tb.asParser -// objectParser.nextToken() -// if (curNode.has("i")) { -// objectParser.readValueAs[B](classOf[B]) -// } else if (curNode.has("s1")) { -// objectParser.readValueAs[C](classOf[C]) -// } else ??? -// } -// } -// ) -// m.addDeserializer( -// classOf[LL], -// new StdDeserializer[LL](classOf[LL]) { -// def deserialize(jp: JsonParser, ctxt: DeserializationContext): LL = { -// val tb = new TokenBuffer(jp, ctxt) -// tb.copyCurrentStructure(jp) -// val firstParser = tb.asParser -// firstParser.nextToken -// val curNode = firstParser.getCodec.readTree[JsonNode](firstParser) -// val objectParser = tb.asParser -// objectParser.nextToken() -// if (curNode.has("c")) { -// objectParser.readValueAs[Node](classOf[Node]) -// } else{ -// End -// } -// } -// } -// ) -// mapper.registerModule(m) -// -// val jacksonType = new TypeReference[Common.Data] {} -// -// Common.bench[String](duration)( -// mapper.readValue[Seq[Common.Data]](_, jacksonType), -// mapper.writeValueAsString(_) -// ) -// } } diff --git a/bench/src-native/Main.scala b/bench/src-native/Main.scala index 2ee6aee57..e56b37c8f 100644 --- a/bench/src-native/Main.scala +++ b/bench/src-native/Main.scala @@ -8,18 +8,48 @@ object Main{ import Hierarchy._ import Recursive._ def main(args: Array[String]): Unit = { - for(duration <- Seq(2500, 5000, 10000, 10000, 10000, 10000)){ + val allResults = collection.mutable.Buffer.empty[(String, Int)] + for ((duration, save) <- Seq(2500 -> false, 5000 -> false, 10000 -> true, 10000 -> true, 10000 -> true)) { println("RUN NATIVE: " + duration) println() - Common.upickleDefault(duration) - Common.upickleDefaultByteArray(duration) - Common.upickleDefaultBinary(duration) + val results = Seq( + Common.upickleDefault(duration), + Common.upickleDefaultByteArray(duration), + Common.upickleDefaultBinary(duration), - Common.upickleDefaultCached(duration) - Common.upickleDefaultByteArrayCached(duration) - Common.upickleDefaultBinaryCached(duration) + Common.integers(duration), + Common.integersByteArray(duration), + Common.integersBinary(duration), + + Common.doubles(duration), + Common.doublesByteArray(duration), + Common.doublesBinary(duration), + + Common.sequences(duration), + Common.sequencesByteArray(duration), + Common.sequencesBinary(duration), + + Common.shortStrings(duration), + Common.shortStringsByteArray(duration), + Common.shortStringsBinary(duration), + + Common.longStrings(duration), + Common.longStringsByteArray(duration), + Common.longStringsBinary(duration), + + Common.unicodeStrings(duration), + Common.unicodeStringsByteArray(duration), + Common.unicodeStringsBinary(duration), + + Common.caseClasses(duration), + Common.caseClassesByteArray(duration), + Common.caseClassesBinary(duration) + ) + println() + if (save) allResults.appendAll(results.flatten) println() } + Common.prettyPrintResults(allResults) } } diff --git a/bench/src/Common.scala b/bench/src/Common.scala index 86fcea174..97ca82aa0 100644 --- a/bench/src/Common.scala +++ b/bench/src/Common.scala @@ -1,5 +1,7 @@ package upickle +import scala.annotation.nowarn + object Common{ import ADTs.ADT0 import Defaults._ @@ -314,6 +316,70 @@ object Common{ // ) // } + + def microBenchDefault[T: upickle.default.ReadWriter](duration: Int, v: T)(implicit n: sourcecode.Name) = { + Common.bench0[String, T](duration, upickle.default.write(v))( + upickle.default.read[T](_), + upickle.default.write(_), + ) + } + + def microBenchDefaultByteArray[T: upickle.default.ReadWriter](duration: Int, v: T)(implicit n: sourcecode.Name) = { + Common.bench0[Array[Byte], T](duration, upickle.default.writeToByteArray(v))( + upickle.default.read[T](_), + upickle.default.writeToByteArray(_), + ) + } + + def microBenchDefaultBinary[T: upickle.default.ReadWriter](duration: Int, v: T)(implicit n: sourcecode.Name) = { + Common.bench0[Array[Byte], T](duration, upickle.default.writeBinary(v))( + upickle.default.readBinary[T](_), + upickle.default.writeBinary(_), + ) + } + + def integers(duration: Int) = microBenchDefault(duration, Micro.integers) + + def doubles(duration: Int) = microBenchDefault(duration, Micro.doubles) + + def sequences(duration: Int) = microBenchDefault(duration, Micro.sequences) + + def shortStrings(duration: Int) = microBenchDefault(duration, Micro.shortStrings) + + def longStrings(duration: Int) = microBenchDefault(duration, Micro.longStrings) + + def unicodeStrings(duration: Int) = microBenchDefault(duration, Micro.unicodeStrings) + + def caseClasses(duration: Int) = microBenchDefault(duration, Micro.nestedCaseClasses) + + def integersByteArray(duration: Int) = microBenchDefaultByteArray(duration, Micro.integers) + + def doublesByteArray(duration: Int) = microBenchDefaultByteArray(duration, Micro.doubles) + + def sequencesByteArray(duration: Int) = microBenchDefaultByteArray(duration, Micro.sequences) + + def shortStringsByteArray(duration: Int) = microBenchDefaultByteArray(duration, Micro.shortStrings) + + def longStringsByteArray(duration: Int) = microBenchDefaultByteArray(duration, Micro.longStrings) + + def unicodeStringsByteArray(duration: Int) = microBenchDefaultByteArray(duration, Micro.unicodeStrings) + + def caseClassesByteArray(duration: Int) = microBenchDefaultByteArray(duration, Micro.nestedCaseClasses) + + def integersBinary(duration: Int) = microBenchDefaultBinary(duration, Micro.integers) + + def doublesBinary(duration: Int) = microBenchDefaultBinary(duration, Micro.doubles) + + def sequencesBinary(duration: Int) = microBenchDefaultBinary(duration, Micro.sequences) + + def shortStringsBinary(duration: Int) = microBenchDefaultBinary(duration, Micro.shortStrings) + + def longStringsBinary(duration: Int) = microBenchDefaultBinary(duration, Micro.longStrings) + + def unicodeStringsBinary(duration: Int) = microBenchDefaultBinary(duration, Micro.unicodeStrings) + + def caseClassesBinary(duration: Int) = microBenchDefaultBinary(duration, Micro.nestedCaseClasses) + def bench[T](duration: Int) (f1: T => Seq[Data], f2: Seq[Data] => T, checkEqual: Boolean = true) (implicit name: sourcecode.Name) = { @@ -335,7 +401,7 @@ object Common{ def bench0[T, V](duration: Int, stringified: T) (f1: T => V, f2: V => T) (implicit name: sourcecode.Name)= { - { + val readResult = { var n = 0 val start = System.currentTimeMillis() while(System.currentTimeMillis() < start + duration){ @@ -343,11 +409,12 @@ object Common{ n += 1 } println(name.value + " Read " + n) + (name.value + " Read ", n) } val parsed = f1(stringified) - { + val writeResult = { var n = 0 val start = System.currentTimeMillis() while(System.currentTimeMillis() < start + duration){ @@ -355,6 +422,31 @@ object Common{ n += 1 } println(name.value + " Write " + n) + (name.value + " Write ", n) } + Seq(readResult, writeResult) + } + + def prettyPrintResults(allResults: collection.Seq[(String, Int)]): String = { + @nowarn("cat=deprecation") + val groupedResults = allResults + .map { + case (s"${n}ByteArray $rw", v) => (s"$n $rw", "JsonByteArray", v) + case (s"${n}Binary $rw", v) => (s"$n $rw", "MsgPack", v) + case (n, v) => (n, "JsonString", v) + } + .groupMap { case (name, tag, v) => (name, tag) }(_._3).mapValues(_.sorted.apply(1 /* get the median of 3*/)) + .groupMap { case ((name, tag), v) => name } { case ((name, tag), v) => (tag, v) }.mapValues(_.toMap) + + val tags = Seq("JsonString", "JsonByteArray", "MsgPack") + val lines = Seq( + "| Name | " + tags.mkString(" | ") + " |", + "|---:| " + tags.map(_ => "---:").mkString(" | ") + " |" + ) ++ ( + for ((name, taggedValues) <- groupedResults.toList.sortBy(t => allResults.map(_._1).indexOf(t._1))) + yield s"| $name | " + tags.map(taggedValues.getOrElse(_, " ")).mkString(" | ") + " |" + ) + + lines.mkString("\n") } } diff --git a/bench/src/Micro.scala b/bench/src/Micro.scala new file mode 100644 index 000000000..88c063fc0 --- /dev/null +++ b/bench/src/Micro.scala @@ -0,0 +1,55 @@ +package upickle +object Micro { + + val longStrings: Array[String] = Array.fill(1000)( + """Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed + |do eiusmod tempor incididunt ut labore et dolore magna aliqua. + |Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris + |nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in + |reprehenderit in voluptate velit esse cillum dolore eu fugiat 1a + |pariatur. Excepteur sint occaecat cupidatat non proident, sunt in + |culpa qui officia deserunt mollit anim id est laborum.""".stripMargin + ) + val shortStrings: Array[String] = Array.tabulate(1000)(i => "hello " + i) + val unicodeStrings: Array[String] = Array.fill(1000)( + "虾饺 烧卖 凤爪 糯米鸡 萝卜糕 小笼包 肠粉 叉烧包 莲蓉包 流沙包 蛋挞 春卷 锅贴" + ) + type NestedADT = Generic.ADT[ + Generic.ADT[Boolean, Boolean, Boolean, Boolean, Boolean, Boolean], + Generic.ADT[Boolean, Boolean, Boolean, Boolean, Boolean, Boolean], + Generic.ADT[Boolean, Boolean, Boolean, Boolean, Boolean, Boolean], + Generic.ADT[Boolean, Boolean, Boolean, Boolean, Boolean, Boolean], + Generic.ADT[Boolean, Boolean, Boolean, Boolean, Boolean, Boolean], + Generic.ADT[Boolean, Boolean, Boolean, Boolean, Boolean, Boolean] + ] + val nestedCaseClasses: Seq[NestedADT] = + Seq.tabulate(1000) { i => + val b = i % 2 == 0 + Generic.ADT( + Generic.ADT(b, b, b, b, b, b), + Generic.ADT(b, b, b, b, b, b), + Generic.ADT(b, b, b, b, b, b), + Generic.ADT(b, b, b, b, b, b), + Generic.ADT(b, b, b, b, b, b), + Generic.ADT(b, b, b, b, b, b) + ) + } + + val sealedTraits: Seq[DeepHierarchy.A] = Seq.tabulate(1000) { i => + i % 3 match{ + case 0 => DeepHierarchy.B(1) + case 1 => DeepHierarchy.D("") + case 2 => DeepHierarchy.E(true) + } + } + + val integers = Seq.tabulate(1000)(n => + n * 123456789 + ) + val doubles = Seq.tabulate(1000)(n => + n * 1234.56789 + ) + val sequences = Seq.tabulate(1000)(n => + Seq.fill(n)(true) + ) +} diff --git a/build.sc b/build.sc index 0f6496629..45531ab5a 100644 --- a/build.sc +++ b/build.sc @@ -66,7 +66,7 @@ trait CommonModule extends CommonBaseModule { trait CommonPublishModule extends CommonModule with PublishModule with Mima with CrossScalaModule { def publishVersion = VcsVersion.vcsState().format() - override def mimaPreviousVersions = Seq("3.0.0-M2") + override def mimaPreviousVersions = Seq("3.0.0") def isDotty = crossScalaVersion.startsWith("0") || crossScalaVersion.startsWith("3") def pomSettings = PomSettings( description = artifactName(), @@ -83,15 +83,16 @@ trait CommonPublishModule extends CommonModule with PublishModule with Mima with "info.releaseNotesURL" -> "https://com-lihaoyi.github.io/upickle/#VersionHistory" ) override def versionScheme: T[Option[VersionScheme]] = T(Some(VersionScheme.SemVerSpec)) - def templates = T.source(millSourcePath / "templates") + def templates = T.sources(millSourcePath / "templates", millSourcePath / s"templates-$platformSegment") override def generatedSources = T{ for{ - p <- if (os.exists(templates().path)) os.list(templates().path) else Nil + pathRef <- templates() + p <- if (os.exists(pathRef.path)) os.list(pathRef.path) else Nil rename <- Seq("Char", "Byte") }{ os.write( - T.dest / p.last.replace("Elem", rename), - os.read(p).replace("Elem", rename) + T.dest / p.last.replace("Elem", rename).replace("elem", rename.toLowerCase), + os.read(p).replace("Elem", rename).replace("elem", rename.toLowerCase) ) } Seq(PathRef(T.dest)) @@ -416,6 +417,9 @@ object upickle extends Module{ Seq("-Dfile.encoding=US-ASCII") } } + object testSlow extends Tests with CommonTestModule{ + def moduleDeps = super.moduleDeps ++ Seq(JvmModule.this.test) + } } object js extends Cross[JsModule](scalaJSVersions:_*) diff --git a/upickle/testSlow/src-js-native/upickle/withTimeout.scala b/upickle/testSlow/src-js-native/upickle/withTimeout.scala new file mode 100644 index 000000000..2312f134b --- /dev/null +++ b/upickle/testSlow/src-js-native/upickle/withTimeout.scala @@ -0,0 +1,9 @@ +package upickle + +object withTimeout { + // In Scala Native and Scala.js it's impossible to + // timeout a blocking operation. So we skip the timeout + def apply(f: => Unit): Unit = { + f + } +} diff --git a/upickle/testSlow/src-jvm/upickle/withTimeout.scala b/upickle/testSlow/src-jvm/upickle/withTimeout.scala new file mode 100644 index 000000000..508064851 --- /dev/null +++ b/upickle/testSlow/src-jvm/upickle/withTimeout.scala @@ -0,0 +1,16 @@ +package upickle + +import scala.concurrent._ +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ + +object withTimeout { + def apply(f: => Unit): Unit = { + Await.result( + Future { + blocking { f } + }, + 2.minutes + ) + } +} diff --git a/upickle/testSlow/src/upickle/SlowTests.scala b/upickle/testSlow/src/upickle/SlowTests.scala new file mode 100644 index 000000000..95c316d18 --- /dev/null +++ b/upickle/testSlow/src/upickle/SlowTests.scala @@ -0,0 +1,121 @@ +package upickle + +import utest._ +import upickle.default.{ReadWriter, macroRW} +object SlowTests extends TestSuite { + val tests = Tests { + test("hash-collision") { + def zeroHashCodeStrings: Iterator[String] = { + def charAndHash(h: Int): Iterator[(Char, Int)] = ('!' to '~').iterator.map(ch => (ch, (h + ch) * 31)) + + for { + (ch0, h0) <- charAndHash(0) + (ch1, h1) <- charAndHash(h0) + (ch2, h2) <- charAndHash(h1) if ((h2 + 32) * 923521 ^ (h2 + 127) * 923521) < 0 + (ch3, h3) <- charAndHash(h2) if ((h3 + 32) * 29791 ^ (h3 + 127) * 29791) < 0 + (ch4, h4) <- charAndHash(h3) if ((h4 + 32) * 961 ^ (h4 + 127) * 961) < 0 + (ch5, h5) <- charAndHash(h4) if ((h5 + 32) * 31 ^ (h5 + 127) * 31) < 0 + (ch6, h6) <- charAndHash(h5) if (h6 + 32 ^ h6 + 127) < 0 + (ch7, _) <- charAndHash(h6) if h6 + ch7 == 0 + } yield new String(Array(ch0, ch1, ch2, ch3, ch4, ch5, ch6, ch7)) + } + + val jsonString = + zeroHashCodeStrings + .map(s => ujson.write(s)) + .take(1000000) + .mkString("{", s":null,", ":null}") + + test("issue-416") { + withTimeout { + upickle.default.read[Foo416](jsonString) + } + } + test("issue-446") { + sys.props("java.vm.name") match { + case "Scala.js" | "Scala Native" => + // The fix assumes a hash collision safe java.util.LinkedHashMap + // implementation. When/if other platforms will have such characteristics + // ujson doesn't need to change. + case _ => + withTimeout { + ujson.read(jsonString) + } + } + () + } + } + test("fuzz") { + test("string") { + test("ascii") - { + for (i <- Range(0, 4096)) { + TestUtil.rw("x" * i, "\"" + ("x" * i) + "\"", upack.Str("x" * i)) + } + } + test("escaped") - { + for (i <- Range(0, 4096)) { + TestUtil.rw("\"" * i, "\"" + ("\\\"" * i) + "\"", upack.Str("\"" * i)) + } + } + test("unicode") - { + for (i <- Range(0, 4096)) { + TestUtil.rw("包" * i, "\"" + ("包" * i) + "\"", upack.Str("包" * i)) + } + } + test("asciiUnicode") - { + for (i <- Range(0, 4096)) { + TestUtil.rw("x包" * i, "\"" + "x包" * i + "\"", upack.Str("x包" * i)) + } + } + test("unicodeEscaped") - { + for (i <- Range(0, 4096)) { + TestUtil.rw("包\"" * i, "\"" + "包\\\"" * i + "\"", upack.Str("包\"" * i)) + } + } + test("escapedAscii") - { + for (i <- Range(0, 4192)) { + val s = + TestUtil.rw("\"x" * i, "\"" + "\\\"x" * i + "\"", upack.Str("\"x" * i)) + } + } + } + test("int"){ + test("topLevel"){ + for (i <- Range(Int.MinValue, Int.MaxValue, 13373)) { + TestUtil.rw(i, s"$i", upack.Int32(i)) + } + } + test("nested"){ + for (i <- Range(Int.MinValue, Int.MaxValue, 13373)) { + TestUtil.rw(Seq(i), s"[$i]", upack.Arr(upack.Int32(i))) + } + } + } + test("long"){ + test("topLevel"){ + for (i <- Range.Long(Long.MinValue, Long.MaxValue, 337133713371337L)) { + TestUtil.rw( + i, + if (math.abs(i) > math.pow(2, 53) || i == Long.MinValue) "\"" + i + "\"" else s"$i" + ) + } + } + test("nested"){ + for (i <- Range.Long(Long.MinValue, Long.MaxValue, 337133713371337L)) { + TestUtil.rw( + Seq(i), + if (math.abs(i) > math.pow(2, 53) || i == Long.MinValue) "[\"" + i + "\"]" else s"[$i]" + ) + } + } + } + } + } +} + + +case class Foo416() + +object Foo416 { + implicit val rw: ReadWriter[Foo416] = macroRW[Foo416] +} From 8e193dd80679a615732e348b09b883192c063bd9 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 14:11:57 +0800 Subject: [PATCH 02/17] fast double parser and BaseElemRenderer.scala updates --- core/src/upickle/core/RenderUtils.scala | 35 + core/templates/ElemBuilder.scala | 4 +- ujson/src-jvm/ujson/MathUtils.java | 826 +++++++++++++++++++ ujson/templates-js/ToDecimalElem.scala | 24 + ujson/templates-jvm/DoubleToDecimalElem.java | 588 +++++++++++++ ujson/templates-jvm/FloatToDecimalElem.java | 560 +++++++++++++ ujson/templates-native/ToDecimalElem.scala | 24 + ujson/templates/BaseElemRenderer.scala | 285 ++++++- upickle/test/src/upickle/AdvancedTests.scala | 42 - 9 files changed, 2308 insertions(+), 80 deletions(-) create mode 100644 ujson/src-jvm/ujson/MathUtils.java create mode 100644 ujson/templates-js/ToDecimalElem.scala create mode 100644 ujson/templates-jvm/DoubleToDecimalElem.java create mode 100644 ujson/templates-jvm/FloatToDecimalElem.java create mode 100644 ujson/templates-native/ToDecimalElem.scala diff --git a/core/src/upickle/core/RenderUtils.scala b/core/src/upickle/core/RenderUtils.scala index 6a4506326..eb486db76 100644 --- a/core/src/upickle/core/RenderUtils.scala +++ b/core/src/upickle/core/RenderUtils.scala @@ -152,4 +152,39 @@ object RenderUtils{ sb.ensureLength(naiveOutLen - i + 1); sb.appendUnsafeC('\\'); sb.appendUnsafeC(c) } + def intStringSize(x0: Int): Int = { + // Taken from java.lang.Int.stringSize + var x = x0 + var d = 1 + if (x >= 0) { + d = 0 + x = -x + } + var p = -10 + var i = 1 + while (i <= 10) { + if (x > p) return i + d + p = 10 * p + i += 1 + } + 10 + d + } + + def longStringSize(x0: Long): Int = { + // Taken from java.lang.Long.stringSize + var x = x0 + var d = 1 + if (x >= 0) { + d = 0 + x = -x + } + var p: Long = -10 + var i = 1 + while (i <= 18) { + if (x > p) return i + d + p = 10 * p + i += 1 + } + 19 + d + } } diff --git a/core/templates/ElemBuilder.scala b/core/templates/ElemBuilder.scala index f8606e858..1faa62faf 100644 --- a/core/templates/ElemBuilder.scala +++ b/core/templates/ElemBuilder.scala @@ -14,8 +14,8 @@ package upickle.core * let you push the data elsewhere with minimal unnecessary copying */ class ElemBuilder(startSize: Int = 32) extends upickle.core.ElemAppendC{ - private[this] var arr: Array[Elem] = new Array(startSize) - private[this] var length: Int = 0 + var arr: Array[Elem] = new Array(startSize) + var length: Int = 0 private def getArr = arr def getLength = length def reset(): Unit = length = 0 diff --git a/ujson/src-jvm/ujson/MathUtils.java b/ujson/src-jvm/ujson/MathUtils.java new file mode 100644 index 000000000..9fde8cb7b --- /dev/null +++ b/ujson/src-jvm/ujson/MathUtils.java @@ -0,0 +1,826 @@ +// Vendored from Jackson +// https://github.com/FasterXML/jackson-core/blob/d26e01fbb4a47354f3c956abe6b120340030aad2/src/main/java/com/fasterxml/jackson/core/io/schubfach/MathUtils.java +/* + * Copyright 2018-2020 Raffaello Giulietti + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package ujson; + +/** + * This class exposes package private utilities for other classes. + * Thus, all methods are assumed to be invoked with correct arguments, + * so these are not checked at all. + * + * @author Raffaello Giulietti + */ +final class MathUtils { + /* + For full details about this code see the following reference: + Giulietti, "The Schubfach way to render doubles", + https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN + */ + + // Sources with the license are here: https://github.com/c4f7fcce9cb06515/Schubfach/blob/3c92d3c9b1fead540616c918cdfef432bca53dfa/todec/src/math/FloatToDecimal.java + + /* + The boundaries for k in g0(int) and g1(int). + K_MIN must be DoubleToDecimal.K_MIN or less. + K_MAX must be DoubleToDecimal.K_MAX or more. + */ + static final int K_MIN = -324; + static final int K_MAX = 292; + + // Must be DoubleToDecimal.H or more + static final int H = 17; + + // C_10 = floor(log10(2) * 2^Q_10), A_10 = floor(log10(3/4) * 2^Q_10) + private static final int Q_10 = 41; + private static final long C_10 = 661_971_961_083L; + private static final long A_10 = -274_743_187_321L; + + // C_2 = floor(log2(10) * 2^Q_2) + private static final int Q_2 = 38; + private static final long C_2 = 913_124_641_741L; + + private MathUtils() { + } + + // The first powers of 10. The last entry must be 10^H. + private static final long[] pow10 = { + 1L, + 10L, + 100L, + 1_000L, + 10_000L, + 100_000L, + 1_000_000L, + 10_000_000L, + 100_000_000L, + 1_000_000_000L, + 10_000_000_000L, + 100_000_000_000L, + 1_000_000_000_000L, + 10_000_000_000_000L, + 100_000_000_000_000L, + 1_000_000_000_000_000L, + 10_000_000_000_000_000L, + 100_000_000_000_000_000L, + }; + + /** + * Returns 10{@code e}. + * + * @param e The exponent which must meet + * 0 ≤ {@code e} ≤ {@link #H}. + * @return 10{@code e}. + */ + static long pow10(int e) { + return pow10[e]; + } + + /** + * Returns the unique integer k such that + * 10k ≤ 2{@code e} + * < 10k+1. + *

+ * The result is correct when |{@code e}| ≤ 5_456_721. + * Otherwise the result is undefined. + * + * @param e The exponent of 2, which should meet + * |{@code e}| ≤ 5_456_721 for safe results. + * @return ⌊log102{@code e}⌋. + */ + static int flog10pow2(int e) { + return (int) (e * C_10 >> Q_10); + } + + /** + * Returns the unique integer k such that + * 10k ≤ 3/4 · 2{@code e} + * < 10k+1. + *

+ * The result is correct when + * -2_956_395 ≤ {@code e} ≤ 2_500_325. + * Otherwise the result is undefined. + * + * @param e The exponent of 2, which should meet + * -2_956_395 ≤ {@code e} ≤ 2_500_325 for safe results. + * @return ⌊log10(3/4 · + * 2{@code e})⌋. + */ + static int flog10threeQuartersPow2(int e) { + return (int) (e * C_10 + A_10 >> Q_10); + } + + /** + * Returns the unique integer k such that + * 2k ≤ 10{@code e} + * < 2k+1. + *

+ * The result is correct when |{@code e}| ≤ 1_838_394. + * Otherwise the result is undefined. + * + * @param e The exponent of 10, which should meet + * |{@code e}| ≤ 1_838_394 for safe results. + * @return ⌊log210{@code e}⌋. + */ + static int flog2pow10(int e) { + return (int) (e * C_2 >> Q_2); + } + + /** + * Let 10-{@code k} = β 2r, + * for the unique pair of integer r and real β meeting + * 2125 ≤ β < 2126. + * Further, let g = ⌊β⌋ + 1. + * Split g into the higher 63 bits g1 and + * the lower 63 bits g0. Thus, + * g1 = + * ⌊g 2-63⌋ + * and + * g0 = + * g - g1 263. + *

+ * This method returns g1 while + * {@link #g0(int)} returns g0. + *

+ * If needed, the exponent r can be computed as + * r = {@code flog2pow10(-k)} - 125 (see {@link #flog2pow10(int)}). + * + * @param k The exponent of 10, which must meet + * {@link #K_MIN} ≤ {@code e} ≤ {@link #K_MAX}. + * @return g1 as described above. + */ + static long g1(int k) { + return g[k - K_MIN << 1]; + } + + /** + * Returns g0 as described in + * {@link #g1(int)}. + * + * @param k The exponent of 10, which must meet + * {@link #K_MIN} ≤ {@code e} ≤ {@link #K_MAX}. + * @return g0 as described in + * {@link #g1(int)}. + */ + static long g0(int k) { + return g[k - K_MIN << 1 | 1]; + } + + //a Java port of https://github.com/plokhotnyuk/jsoniter-scala/blob/c70a293ac802dc2eb44165471d76d7df2d4657b6/jsoniter-scala-core/native/src/main/scala/com/github/plokhotnyuk/jsoniter_scala/core/JsonWriter.scala#L2027 + static long multiplyHigh(long x, long y) { + // Karatsuba technique for two positive ints + long x2 = x & 0xFFFFFFFFL; + long y2 = y & 0xFFFFFFFFL; + long b = x2 * y2; + long x1 = x >>> 32; + long y1 = y >>> 32; + long a = x1 * y1; + return (((b >>> 32) + (x1 + x2) * (y1 + y2) - b - a) >>> 32) + a; + } + + /* + The precomputed values for g1(int) and g0(int). + The first entry must be for an exponent of K_MIN or less. + The last entry must be for an exponent of K_MAX or more. + */ + private static final long[] g = { + /* -324 */ 0x4F0C_EDC9_5A71_8DD4L, 0x5B01_E8B0_9AA0_D1B5L, + /* -323 */ 0x7E7B_160E_F71C_1621L, 0x119C_A780_F767_B5EEL, + /* -322 */ 0x652F_44D8_C5B0_11B4L, 0x0E16_EC67_2C52_F7F2L, + /* -321 */ 0x50F2_9D7A_37C0_0E29L, 0x5812_56B8_F042_5FF5L, + /* -320 */ 0x40C2_1794_F966_71BAL, 0x79A8_4560_C035_1991L, + /* -319 */ 0x679C_F287_F570_B5F7L, 0x75DA_089A_CD21_C281L, + /* -318 */ 0x52E3_F539_9126_F7F9L, 0x44AE_6D48_A41B_0201L, + /* -317 */ 0x424F_F761_40EB_F994L, 0x36F1_F106_E9AF_34CDL, + /* -316 */ 0x6A19_8BCE_CE46_5C20L, 0x57E9_81A4_A918_547BL, + /* -315 */ 0x54E1_3CA5_71D1_E34DL, 0x2CBA_CE1D_5413_76C9L, + /* -314 */ 0x43E7_63B7_8E41_82A4L, 0x23C8_A4E4_4342_C56EL, + /* -313 */ 0x6CA5_6C58_E39C_043AL, 0x060D_D4A0_6B9E_08B0L, + /* -312 */ 0x56EA_BD13_E949_9CFBL, 0x1E71_76E6_BC7E_6D59L, + /* -311 */ 0x4588_9743_2107_B0C8L, 0x7EC1_2BEB_C9FE_BDE1L, + /* -310 */ 0x6F40_F205_01A5_E7A7L, 0x7E01_DFDF_A997_9635L, + /* -309 */ 0x5900_C19D_9AEB_1FB9L, 0x4B34_B319_5479_44F7L, + /* -308 */ 0x4733_CE17_AF22_7FC7L, 0x55C3_C27A_A9FA_9D93L, + /* -307 */ 0x71EC_7CF2_B1D0_CC72L, 0x5606_03F7_765D_C8EAL, + /* -306 */ 0x5B23_9728_8E40_A38EL, 0x7804_CFF9_2B7E_3A55L, + /* -305 */ 0x48E9_45BA_0B66_E93FL, 0x1337_0CC7_55FE_9511L, + /* -304 */ 0x74A8_6F90_123E_41FEL, 0x51F1_AE0B_BCCA_881BL, + /* -303 */ 0x5D53_8C73_41CB_67FEL, 0x74C1_5809_63D5_39AFL, + /* -302 */ 0x4AA9_3D29_016F_8665L, 0x43CD_E007_8310_FAF3L, + /* -301 */ 0x7775_2EA8_024C_0A3CL, 0x0616_333F_381B_2B1EL, + /* -300 */ 0x5F90_F220_01D6_6E96L, 0x3811_C298_F9AF_55B1L, + /* -299 */ 0x4C73_F4E6_67DE_BEDEL, 0x600E_3547_2E25_DE28L, + /* -298 */ 0x7A53_2170_A631_3164L, 0x3349_EED8_49D6_303FL, + /* -297 */ 0x61DC_1AC0_84F4_2783L, 0x42A1_8BE0_3B11_C033L, + /* -296 */ 0x4E49_AF00_6A5C_EC69L, 0x1BB4_6FE6_95A7_CCF5L, + /* -295 */ 0x7D42_B19A_43C7_E0A8L, 0x2C53_E63D_BC3F_AE55L, + /* -294 */ 0x6435_5AE1_CFD3_1A20L, 0x2376_51CA_FCFF_BEAAL, + /* -293 */ 0x502A_AF1B_0CA8_E1B3L, 0x35F8_416F_30CC_9888L, + /* -292 */ 0x4022_25AF_3D53_E7C2L, 0x5E60_3458_F3D6_E06DL, + /* -291 */ 0x669D_0918_621F_D937L, 0x4A33_86F4_B957_CD7BL, + /* -290 */ 0x5217_3A79_E819_7A92L, 0x6E8F_9F2A_2DDF_D796L, + /* -289 */ 0x41AC_2EC7_ECE1_2EDBL, 0x720C_7F54_F17F_DFABL, + /* -288 */ 0x6913_7E0C_AE35_17C6L, 0x1CE0_CBBB_1BFF_CC45L, + /* -287 */ 0x540F_980A_24F7_4638L, 0x171A_3C95_AFFF_D69EL, + /* -286 */ 0x433F_ACD4_EA5F_6B60L, 0x127B_63AA_F333_1218L, + /* -285 */ 0x6B99_1487_DD65_7899L, 0x6A5F_05DE_51EB_5026L, + /* -284 */ 0x5614_106C_B11D_FA14L, 0x5518_D17E_A7EF_7352L, + /* -283 */ 0x44DC_D9F0_8DB1_94DDL, 0x2A7A_4132_1FF2_C2A8L, + /* -282 */ 0x6E2E_2980_E2B5_BAFBL, 0x5D90_6850_331E_043FL, + /* -281 */ 0x5824_EE00_B55E_2F2FL, 0x6473_86A6_8F4B_3699L, + /* -280 */ 0x4683_F19A_2AB1_BF59L, 0x36C2_D21E_D908_F87BL, + /* -279 */ 0x70D3_1C29_DDE9_3228L, 0x579E_1CFE_280E_5A5DL, + /* -278 */ 0x5A42_7CEE_4B20_F4EDL, 0x2C7E_7D98_200B_7B7EL, + /* -277 */ 0x4835_30BE_A280_C3F1L, 0x09FE_CAE0_19A2_C932L, + /* -276 */ 0x7388_4DFD_D0CE_064EL, 0x4331_4499_C29E_0EB6L, + /* -275 */ 0x5C6D_0B31_73D8_050BL, 0x4F5A_9D47_CEE4_D891L, + /* -274 */ 0x49F0_D5C1_2979_9DA2L, 0x72AE_E439_7250_AD41L, + /* -273 */ 0x764E_22CE_A8C2_95D1L, 0x377E_39F5_83B4_4868L, + /* -272 */ 0x5EA4_E8A5_53CE_DE41L, 0x12CB_6191_3629_D387L, + /* -271 */ 0x4BB7_2084_430B_E500L, 0x756F_8140_F821_7605L, + /* -270 */ 0x7925_00D3_9E79_6E67L, 0x6F18_CECE_59CF_233CL, + /* -269 */ 0x60EA_670F_B1FA_BEB9L, 0x3F47_0BD8_47D8_E8FDL, + /* -268 */ 0x4D88_5272_F4C8_9894L, 0x329F_3CAD_0647_20CAL, + /* -267 */ 0x7C0D_50B7_EE0D_C0EDL, 0x3765_2DE1_A3A5_0143L, + /* -266 */ 0x633D_DA2C_BE71_6724L, 0x2C50_F181_4FB7_3436L, + /* -265 */ 0x4F64_AE8A_31F4_5283L, 0x3D0D_8E01_0C92_902BL, + /* -264 */ 0x7F07_7DA9_E986_EA6BL, 0x7B48_E334_E0EA_8045L, + /* -263 */ 0x659F_97BB_2138_BB89L, 0x4907_1C2A_4D88_669DL, + /* -262 */ 0x514C_7962_80FA_2FA1L, 0x20D2_7CEE_A46D_1EE4L, + /* -261 */ 0x4109_FAB5_33FB_594DL, 0x670E_CA58_838A_7F1DL, + /* -260 */ 0x680F_F788_532B_C216L, 0x0B4A_DD5A_6C10_CB62L, + /* -259 */ 0x533F_F939_DC23_01ABL, 0x22A2_4AAE_BCDA_3C4EL, + /* -258 */ 0x4299_942E_49B5_9AEFL, 0x354E_A225_63E1_C9D8L, + /* -257 */ 0x6A8F_537D_42BC_2B18L, 0x554A_9D08_9FCF_A95AL, + /* -256 */ 0x553F_75FD_CEFC_EF46L, 0x776E_E406_E63F_BAAEL, + /* -255 */ 0x4432_C4CB_0BFD_8C38L, 0x5F8B_E99F_1E99_6225L, + /* -254 */ 0x6D1E_07AB_4662_79F4L, 0x3279_75CB_6428_9D08L, + /* -253 */ 0x574B_3955_D1E8_6190L, 0x2861_2B09_1CED_4A6DL, + /* -252 */ 0x45D5_C777_DB20_4E0DL, 0x06B4_226D_B0BD_D524L, + /* -251 */ 0x6FBC_7259_5E9A_167BL, 0x2453_6A49_1AC9_5506L, + /* -250 */ 0x5963_8EAD_E548_11FCL, 0x1D0F_883A_7BD4_4405L, + /* -249 */ 0x4782_D88B_1DD3_4196L, 0x4A72_D361_FCA9_D004L, + /* -248 */ 0x726A_F411_C952_028AL, 0x43EA_EBCF_FAA9_4CD3L, + /* -247 */ 0x5B88_C341_6DDB_353BL, 0x4FEF_230C_C887_70A9L, + /* -246 */ 0x493A_35CD_F17C_2A96L, 0x0CBF_4F3D_6D39_26EEL, + /* -245 */ 0x7529_EFAF_E8C6_AA89L, 0x6132_1862_485B_717CL, + /* -244 */ 0x5DBB_2626_53D2_2207L, 0x675B_46B5_06AF_8DFDL, + /* -243 */ 0x4AFC_1E85_0FDB_4E6CL, 0x52AF_6BC4_0559_3E64L, + /* -242 */ 0x77F9_CA6E_7FC5_4A47L, 0x377F_12D3_3BC1_FD6DL, + /* -241 */ 0x5FFB_0858_6637_6E9FL, 0x45FF_4242_9634_CABDL, + /* -240 */ 0x4CC8_D379_EB5F_8BB2L, 0x6B32_9B68_782A_3BCBL, + /* -239 */ 0x7ADA_EBF6_4565_AC51L, 0x2B84_2BDA_59DD_2C77L, + /* -238 */ 0x6248_BCC5_0451_56A7L, 0x3C69_BCAE_AE4A_89F9L, + /* -237 */ 0x4EA0_9704_0374_4552L, 0x6387_CA25_583B_A194L, + /* -236 */ 0x7DCD_BE6C_D253_A21EL, 0x05A6_103B_C05F_68EDL, + /* -235 */ 0x64A4_9857_0EA9_4E7EL, 0x37B8_0CFC_99E5_ED8AL, + /* -234 */ 0x5083_AD12_7221_0B98L, 0x2C93_3D96_E184_BE08L, + /* -233 */ 0x4069_5741_F4E7_3C79L, 0x7075_CADF_1AD0_9807L, + /* -232 */ 0x670E_F203_2171_FA5CL, 0x4D89_4498_2AE7_59A4L, + /* -231 */ 0x5272_5B35_B45B_2EB0L, 0x3E07_6A13_5585_E150L, + /* -230 */ 0x41F5_15C4_9048_F226L, 0x64D2_BB42_AAD1_810DL, + /* -229 */ 0x6988_22D4_1A0E_503EL, 0x07B7_9204_4482_6815L, + /* -228 */ 0x546C_E8A9_AE71_D9CBL, 0x1FC6_0E69_D068_5344L, + /* -227 */ 0x438A_53BA_F1F4_AE3CL, 0x196B_3EBB_0D20_429DL, + /* -226 */ 0x6C10_85F7_E987_7D2DL, 0x0F11_FDF8_1500_6A94L, + /* -225 */ 0x5673_9E5F_EE05_FDBDL, 0x58DB_3193_4400_5543L, + /* -224 */ 0x4529_4B7F_F19E_6497L, 0x60AF_5ADC_3666_AA9CL, + /* -223 */ 0x6EA8_78CC_B5CA_3A8CL, 0x344B_C493_8A3D_DDC7L, + /* -222 */ 0x5886_C70A_2B08_2ED6L, 0x5D09_6A0F_A1CB_17D2L, + /* -221 */ 0x46D2_38D4_EF39_BF12L, 0x173A_BB3F_B4A2_7975L, + /* -220 */ 0x7150_5AEE_4B8F_981DL, 0x0B91_2B99_2103_F588L, + /* -219 */ 0x5AA6_AF25_093F_ACE4L, 0x0940_EFAD_B403_2AD3L, + /* -218 */ 0x4885_58EA_6DCC_8A50L, 0x0767_2624_9002_88A9L, + /* -217 */ 0x7408_8E43_E2E0_DD4CL, 0x723E_A36D_B337_410EL, + /* -216 */ 0x5CD3_A503_1BE7_1770L, 0x5B65_4F8A_F5C5_CDA5L, + /* -215 */ 0x4A42_EA68_E31F_45F3L, 0x62B7_72D5_916B_0AEBL, + /* -214 */ 0x76D1_770E_3832_0986L, 0x0458_B7BC_1BDE_77DDL, + /* -213 */ 0x5F0D_F8D8_2CF4_D46BL, 0x1D13_C630_164B_9318L, + /* -212 */ 0x4C0B_2D79_BD90_A9EFL, 0x30DC_9E8C_DEA2_DC13L, + /* -211 */ 0x79AB_7BF5_FC1A_A97FL, 0x0160_FDAE_3104_9351L, + /* -210 */ 0x6155_FCC4_C9AE_EDFFL, 0x1AB3_FE24_F403_A90EL, + /* -209 */ 0x4DDE_63D0_A158_BE65L, 0x6229_981D_9002_EDA5L, + /* -208 */ 0x7C97_061A_9BC1_30A2L, 0x69DC_2695_B337_E2A1L, + /* -207 */ 0x63AC_04E2_1634_26E8L, 0x54B0_1EDE_28F9_821BL, + /* -206 */ 0x4FBC_D0B4_DE90_1F20L, 0x43C0_18B1_BA61_34E2L, + /* -205 */ 0x7F94_8121_6419_CB67L, 0x1F99_C11C_5D68_549DL, + /* -204 */ 0x6610_674D_E9AE_3C52L, 0x4C7B_00E3_7DED_107EL, + /* -203 */ 0x51A6_B90B_2158_3042L, 0x09FC_00B5_FE57_4065L, + /* -202 */ 0x4152_2DA2_8113_59CEL, 0x3B30_0091_9845_CD1DL, + /* -201 */ 0x6883_7C37_34EB_C2E3L, 0x784C_CDB5_C06F_AE95L, + /* -200 */ 0x539C_635F_5D89_68B6L, 0x2D0A_3E2B_0059_5877L, + /* -199 */ 0x42E3_82B2_B13A_BA2BL, 0x3DA1_CB55_99E1_1393L, + /* -198 */ 0x6B05_9DEA_B52A_C378L, 0x629C_7888_F634_EC1EL, + /* -197 */ 0x559E_17EE_F755_692DL, 0x3549_FA07_2B5D_89B1L, + /* -196 */ 0x447E_798B_F911_20F1L, 0x1107_FB38_EF7E_07C1L, + /* -195 */ 0x6D97_28DF_F4E8_34B5L, 0x01A6_5EC1_7F30_0C68L, + /* -194 */ 0x57AC_20B3_2A53_5D5DL, 0x4E1E_B234_65C0_09EDL, + /* -193 */ 0x4623_4D5C_21DC_4AB1L, 0x24E5_5B5D_1E33_3B24L, + /* -192 */ 0x7038_7BC6_9C93_AAB5L, 0x216E_F894_FD1E_C506L, + /* -191 */ 0x59C6_C96B_B076_222AL, 0x4DF2_6077_30E5_6A6CL, + /* -190 */ 0x47D2_3ABC_8D2B_4E88L, 0x3E5B_805F_5A51_21F0L, + /* -189 */ 0x72E9_F794_1512_1740L, 0x63C5_9A32_2A1B_697FL, + /* -188 */ 0x5BEE_5FA9_AA74_DF67L, 0x0304_7B5B_54E2_BACCL, + /* -187 */ 0x498B_7FBA_EEC3_E5ECL, 0x0269_FC49_10B5_623DL, + /* -186 */ 0x75AB_FF91_7E06_3CACL, 0x6A43_2D41_B455_69FBL, + /* -185 */ 0x5E23_32DA_CB38_308AL, 0x21CF_5767_C377_87FCL, + /* -184 */ 0x4B4F_5BE2_3C2C_F3A1L, 0x67D9_12B9_692C_6CCAL, + /* -183 */ 0x787E_F969_F9E1_85CFL, 0x595B_5128_A847_1476L, + /* -182 */ 0x6065_9454_C7E7_9E3FL, 0x6115_DA86_ED05_A9F8L, + /* -181 */ 0x4D1E_1043_D31F_B1CCL, 0x4DAB_1538_BD9E_2193L, + /* -180 */ 0x7B63_4D39_51CC_4FADL, 0x62AB_5527_95C9_CF52L, + /* -179 */ 0x62B5_D761_0E3D_0C8BL, 0x0222_AA86_116E_3F75L, + /* -178 */ 0x4EF7_DF80_D830_D6D5L, 0x4E82_2204_DABE_992AL, + /* -177 */ 0x7E59_659A_F381_57BCL, 0x1736_9CD4_9130_F510L, + /* -176 */ 0x6514_5148_C2CD_DFC9L, 0x5F5E_E3DD_40F3_F740L, + /* -175 */ 0x50DD_0DD3_CF0B_196EL, 0x1918_B64A_9A5C_C5CDL, + /* -174 */ 0x40B0_D7DC_A5A2_7ABEL, 0x4746_F83B_AEB0_9E3EL, + /* -173 */ 0x6781_5961_0903_F797L, 0x253E_59F9_1780_FD2FL, + /* -172 */ 0x52CD_E11A_6D9C_C612L, 0x50FE_AE60_DF9A_6426L, + /* -171 */ 0x423E_4DAE_BE17_04DBL, 0x5A65_584D_7FAE_B685L, + /* -170 */ 0x69FD_4917_968B_3AF9L, 0x10A2_26E2_65E4_573BL, + /* -169 */ 0x54CA_A0DF_ABA2_9594L, 0x0D4E_8581_EB1D_1295L, + /* -168 */ 0x43D5_4D7F_BC82_1143L, 0x243E_D134_BC17_4211L, + /* -167 */ 0x6C88_7BFF_9403_4ED2L, 0x06CA_E854_6025_3682L, + /* -166 */ 0x56D3_9666_1002_A574L, 0x6BD5_86A9_E684_2B9BL, + /* -165 */ 0x4576_11EB_4002_1DF7L, 0x0977_9EEE_5203_5616L, + /* -164 */ 0x6F23_4FDE_CCD0_2FF1L, 0x5BF2_97E3_B66B_BCEFL, + /* -163 */ 0x58E9_0CB2_3D73_598EL, 0x165B_ACB6_2B89_63F3L, + /* -162 */ 0x4720_D6F4_FDF5_E13EL, 0x4516_23C4_EFA1_1CC2L, + /* -161 */ 0x71CE_24BB_2FEF_CECAL, 0x3B56_9FA1_7F68_2E03L, + /* -160 */ 0x5B0B_5095_BFF3_0BD5L, 0x15DE_E61A_CC53_5803L, + /* -159 */ 0x48D5_DA11_665C_0977L, 0x2B18_B815_7042_ACCFL, + /* -158 */ 0x7489_5CE8_A3C6_758BL, 0x5E8D_F355_806A_AE18L, + /* -157 */ 0x5D3A_B0BA_1C9E_C46FL, 0x653E_5C44_66BB_BE7AL, + /* -156 */ 0x4A95_5A2E_7D4B_D059L, 0x3765_169D_1EFC_9861L, + /* -155 */ 0x7755_5D17_2EDF_B3C2L, 0x256E_8A94_FE60_F3CFL, + /* -154 */ 0x5F77_7DAC_257F_C301L, 0x6ABE_D543_FEB3_F63FL, + /* -153 */ 0x4C5F_97BC_EACC_9C01L, 0x3BCB_DDCF_FEF6_5E99L, + /* -152 */ 0x7A32_8C61_77AD_C668L, 0x5FAC_9619_97F0_975BL, + /* -151 */ 0x61C2_09E7_92F1_6B86L, 0x7FBD_44E1_465A_12AFL, + /* -150 */ 0x4E34_D4B9_425A_BC6BL, 0x7FCA_9D81_0514_DBBFL, + /* -149 */ 0x7D21_545B_9D5D_FA46L, 0x32DD_C8CE_6E87_C5FFL, + /* -148 */ 0x641A_A9E2_E44B_2E9EL, 0x5BE4_A0A5_2539_6B32L, + /* -147 */ 0x5015_54B5_836F_587EL, 0x7CB6_E6EA_842D_EF5CL, + /* -146 */ 0x4011_1091_35F2_AD32L, 0x3092_5255_368B_25E3L, + /* -145 */ 0x6681_B41B_8984_4850L, 0x4DB6_EA21_F0DE_A304L, + /* -144 */ 0x5201_5CE2_D469_D373L, 0x57C5_881B_2718_826AL, + /* -143 */ 0x419A_B0B5_76BB_0F8FL, 0x5FD1_39AF_527A_01EFL, + /* -142 */ 0x68F7_8122_5791_B27FL, 0x4C81_F5E5_50C3_364AL, + /* -141 */ 0x53F9_341B_7941_5B99L, 0x239B_2B1D_DA35_C508L, + /* -140 */ 0x432D_C349_2DCD_E2E1L, 0x02E2_88E4_AE91_6A6DL, + /* -139 */ 0x6B7C_6BA8_4949_6B01L, 0x516A_74A1_174F_10AEL, + /* -138 */ 0x55FD_22ED_076D_EF34L, 0x4121_F6E7_45D8_DA25L, + /* -137 */ 0x44CA_8257_3924_BF5DL, 0x1A81_9252_9E47_14EBL, + /* -136 */ 0x6E10_D08B_8EA1_322EL, 0x5D9C_1D50_FD3E_87DDL, + /* -135 */ 0x580D_73A2_D880_F4F2L, 0x17B0_1773_FDCB_9FE4L, + /* -134 */ 0x4671_294F_139A_5D8EL, 0x4626_7929_97D6_1984L, + /* -133 */ 0x70B5_0EE4_EC2A_2F4AL, 0x3D0A_5B75_BFBC_F59FL, + /* -132 */ 0x5A2A_7250_BCEE_8C3BL, 0x4A6E_AF91_6630_C47FL, + /* -131 */ 0x4821_F50D_63F2_09C9L, 0x21F2_260D_EB5A_36CCL, + /* -130 */ 0x7369_8815_6CB6_760EL, 0x6983_7016_455D_247AL, + /* -129 */ 0x5C54_6CDD_F091_F80BL, 0x6E02_C011_D117_5062L, + /* -128 */ 0x49DD_23E4_C074_C66FL, 0x719B_CCDB_0DAC_404EL, + /* -127 */ 0x762E_9FD4_6721_3D7FL, 0x68F9_47C4_E2AD_33B0L, + /* -126 */ 0x5E8B_B310_5280_FDFFL, 0x6D94_396A_4EF0_F627L, + /* -125 */ 0x4BA2_F5A6_A867_3199L, 0x3E10_2DEE_A58D_91B9L, + /* -124 */ 0x7904_BC3D_DA3E_B5C2L, 0x3019_E317_6F48_E927L, + /* -123 */ 0x60D0_9697_E1CB_C49BL, 0x4014_B5AC_5907_20ECL, + /* -122 */ 0x4D73_ABAC_B4A3_03AFL, 0x4CDD_5E23_7A6C_1A57L, + /* -121 */ 0x7BEC_45E1_2104_D2B2L, 0x47C8_969F_2A46_908AL, + /* -120 */ 0x6323_6B1A_80D0_A88EL, 0x6CA0_787F_5505_406FL, + /* -119 */ 0x4F4F_88E2_00A6_ED3FL, 0x0A19_F9FF_7737_66BFL, + /* -118 */ 0x7EE5_A7D0_010B_1531L, 0x5CF6_5CCB_F1F2_3DFEL, + /* -117 */ 0x6584_8640_00D5_AA8EL, 0x172B_7D6F_F4C1_CB32L, + /* -116 */ 0x5136_D1CC_CD77_BBA4L, 0x78EF_978C_C3CE_3C28L, + /* -115 */ 0x40F8_A7D7_0AC6_2FB7L, 0x13F2_DFA3_CFD8_3020L, + /* -114 */ 0x67F4_3FBE_77A3_7F8BL, 0x3984_9906_1959_E699L, + /* -113 */ 0x5329_CC98_5FB5_FFA2L, 0x6136_E0D1_ADE1_8548L, + /* -112 */ 0x4287_D6E0_4C91_994FL, 0x00F8_B3DA_F181_376DL, + /* -111 */ 0x6A72_F166_E0E8_F54BL, 0x1B27_862B_1C01_F247L, + /* -110 */ 0x5528_C11F_1A53_F76FL, 0x2F52_D1BC_1667_F506L, + /* -109 */ 0x4420_9A7F_4843_2C59L, 0x0C42_4163_451F_F738L, + /* -108 */ 0x6D00_F732_0D38_46F4L, 0x7A03_9BD2_0833_2526L, + /* -107 */ 0x5733_F8F4_D760_38C3L, 0x7B36_1641_A028_EA85L, + /* -106 */ 0x45C3_2D90_AC4C_FA36L, 0x2F5E_7834_8020_BB9EL, + /* -105 */ 0x6F9E_AF4D_E07B_29F0L, 0x4BCA_59ED_99CD_F8FCL, + /* -104 */ 0x594B_BF71_8062_87F3L, 0x563B_7B24_7B0B_2D96L, + /* -103 */ 0x476F_CC5A_CD1B_9FF6L, 0x11C9_2F50_626F_57ACL, + /* -102 */ 0x724C_7A2A_E1C5_CCBDL, 0x02DB_7EE7_03E5_5912L, + /* -101 */ 0x5B70_61BB_E7D1_7097L, 0x1BE2_CBEC_031D_E0DCL, + /* -100 */ 0x4926_B496_530D_F3ACL, 0x164F_0989_9C17_E716L, + /* -99 */ 0x750A_BA8A_1E7C_B913L, 0x3D4B_4275_C68C_A4F0L, + /* -98 */ 0x5DA2_2ED4_E530_940FL, 0x4AA2_9B91_6BA3_B726L, + /* -97 */ 0x4AE8_2577_1DC0_7672L, 0x6EE8_7C74_561C_9285L, + /* -96 */ 0x77D9_D58B_62CD_8A51L, 0x3173_FA53_BCFA_8408L, + /* -95 */ 0x5FE1_77A2_B571_3B74L, 0x278F_FB76_30C8_69A0L, + /* -94 */ 0x4CB4_5FB5_5DF4_2F90L, 0x1FA6_62C4_F3D3_87B3L, + /* -93 */ 0x7ABA_32BB_C986_B280L, 0x32A3_D13B_1FB8_D91FL, + /* -92 */ 0x622E_8EFC_A138_8ECDL, 0x0EE9_742F_4C93_E0E6L, + /* -91 */ 0x4E8B_A596_E760_723DL, 0x58BA_C359_0A0F_E71EL, + /* -90 */ 0x7DAC_3C24_A567_1D2FL, 0x412A_D228_1019_71C9L, + /* -89 */ 0x6489_C9B6_EAB8_E426L, 0x00EF_0E86_7347_8E3BL, + /* -88 */ 0x506E_3AF8_BBC7_1CEBL, 0x1A58_D86B_8F6C_71C9L, + /* -87 */ 0x4058_2F2D_6305_B0BCL, 0x1513_E056_0C56_C16EL, + /* -86 */ 0x66F3_7EAF_04D5_E793L, 0x3B53_0089_AD57_9BE2L, + /* -85 */ 0x525C_6558_D0AB_1FA9L, 0x15DC_006E_2446_164FL, + /* -84 */ 0x41E3_8447_0D55_B2EDL, 0x5E49_99F1_B69E_783FL, + /* -83 */ 0x696C_06D8_1555_EB15L, 0x7D42_8FE9_2430_C065L, + /* -82 */ 0x5456_6BE0_1111_88DEL, 0x3102_0CBA_835A_3384L, + /* -81 */ 0x4378_564C_DA74_6D7EL, 0x5A68_0A2E_CF7B_5C69L, + /* -80 */ 0x6BF3_BD47_C3ED_7BFDL, 0x770C_DD17_B25E_FA42L, + /* -79 */ 0x565C_976C_9CBD_FCCBL, 0x1270_B0DF_C1E5_9502L, + /* -78 */ 0x4516_DF8A_16FE_63D5L, 0x5B8D_5A4C_9B1E_10CEL, + /* -77 */ 0x6E8A_FF43_57FD_6C89L, 0x127B_C3AD_C4FC_E7B0L, + /* -76 */ 0x586F_329C_4664_56D4L, 0x0EC9_6957_D0CA_52F3L, + /* -75 */ 0x46BF_5BB0_3850_4576L, 0x3F07_8779_73D5_0F29L, + /* -74 */ 0x7132_2C4D_26E6_D58AL, 0x31A5_A58F_1FBB_4B75L, + /* -73 */ 0x5A8E_89D7_5252_446EL, 0x5AEA_EAD8_E62F_6F91L, + /* -72 */ 0x4872_07DF_750E_9D25L, 0x2F22_557A_51BF_8C74L, + /* -71 */ 0x73E9_A632_54E4_2EA2L, 0x1836_EF2A_1C65_AD86L, + /* -70 */ 0x5CBA_EB5B_771C_F21BL, 0x2CF8_BF54_E384_8AD2L, + /* -69 */ 0x4A2F_22AF_927D_8E7CL, 0x23FA_32AA_4F9D_3BDBL, + /* -68 */ 0x76B1_D118_EA62_7D93L, 0x5329_EAAA_18FB_92F8L, + /* -67 */ 0x5EF4_A747_21E8_6476L, 0x0F54_BBBB_472F_A8C6L, + /* -66 */ 0x4BF6_EC38_E7ED_1D2BL, 0x25DD_62FC_38F2_ED6CL, + /* -65 */ 0x798B_138E_3FE1_C845L, 0x22FB_D193_8E51_7BDFL, + /* -64 */ 0x613C_0FA4_FFE7_D36AL, 0x4F2F_DADC_71DA_C97FL, + /* -63 */ 0x4DC9_A61D_9986_42BBL, 0x58F3_157D_27E2_3ACCL, + /* -62 */ 0x7C75_D695_C270_6AC5L, 0x74B8_2261_D969_F7ADL, + /* -61 */ 0x6391_7877_CEC0_556BL, 0x1093_4EB4_ADEE_5FBEL, + /* -60 */ 0x4FA7_9393_0BCD_1122L, 0x4075_D890_8B25_1965L, + /* -59 */ 0x7F72_85B8_12E1_B504L, 0x00BC_8DB4_11D4_F56EL, + /* -58 */ 0x65F5_37C6_7581_5D9CL, 0x66FD_3E29_A7DD_9125L, + /* -57 */ 0x5190_F96B_9134_4AE3L, 0x6BFD_CB54_864A_DA84L, + /* -56 */ 0x4140_C789_40F6_A24FL, 0x6FFE_3C43_9EA2_486AL, + /* -55 */ 0x6867_A5A8_67F1_03B2L, 0x7FFD_2D38_FDD0_73DCL, + /* -54 */ 0x5386_1E20_5327_3628L, 0x6664_242D_97D9_F64AL, + /* -53 */ 0x42D1_B1B3_75B8_F820L, 0x51E9_B68A_DFE1_91D5L, + /* -52 */ 0x6AE9_1C52_55F4_C034L, 0x1CA9_2411_6635_B621L, + /* -51 */ 0x5587_49DB_77F7_0029L, 0x63BA_8341_1E91_5E81L, + /* -50 */ 0x446C_3B15_F992_6687L, 0x6962_029A_7EDA_B201L, + /* -49 */ 0x6D79_F823_28EA_3DA6L, 0x0F03_375D_97C4_5001L, + /* -48 */ 0x5794_C682_8721_CAEBL, 0x259C_2C4A_DFD0_4001L, + /* -47 */ 0x4610_9ECE_D281_6F22L, 0x5149_BD08_B30D_0001L, + /* -46 */ 0x701A_97B1_50CF_1837L, 0x3542_C80D_EB48_0001L, + /* -45 */ 0x59AE_DFC1_0D72_79C5L, 0x7768_A00B_22A0_0001L, + /* -44 */ 0x47BF_1967_3DF5_2E37L, 0x7920_8008_E880_0001L, + /* -43 */ 0x72CB_5BD8_6321_E38CL, 0x5B67_3341_7400_0001L, + /* -42 */ 0x5BD5_E313_8281_82D6L, 0x7C52_8F67_9000_0001L, + /* -41 */ 0x4977_E8DC_6867_9BDFL, 0x16A8_72B9_4000_0001L, + /* -40 */ 0x758C_A7C7_0D72_92FEL, 0x5773_EAC2_0000_0001L, + /* -39 */ 0x5E0A_1FD2_7128_7598L, 0x45F6_5568_0000_0001L, + /* -38 */ 0x4B3B_4CA8_5A86_C47AL, 0x04C5_1120_0000_0001L, + /* -37 */ 0x785E_E10D_5DA4_6D90L, 0x07A1_B500_0000_0001L, + /* -36 */ 0x604B_E73D_E483_8AD9L, 0x52E7_C400_0000_0001L, + /* -35 */ 0x4D09_85CB_1D36_08AEL, 0x0F1F_D000_0000_0001L, + /* -34 */ 0x7B42_6FAB_61F0_0DE3L, 0x31CC_8000_0000_0001L, + /* -33 */ 0x629B_8C89_1B26_7182L, 0x5B0A_0000_0000_0001L, + /* -32 */ 0x4EE2_D6D4_15B8_5ACEL, 0x7C08_0000_0000_0001L, + /* -31 */ 0x7E37_BE20_22C0_914BL, 0x1340_0000_0000_0001L, + /* -30 */ 0x64F9_64E6_8233_A76FL, 0x2900_0000_0000_0001L, + /* -29 */ 0x50C7_83EB_9B5C_85F2L, 0x5400_0000_0000_0001L, + /* -28 */ 0x409F_9CBC_7C4A_04C2L, 0x1000_0000_0000_0001L, + /* -27 */ 0x6765_C793_FA10_079DL, 0x0000_0000_0000_0001L, + /* -26 */ 0x52B7_D2DC_C80C_D2E4L, 0x0000_0000_0000_0001L, + /* -25 */ 0x422C_A8B0_A00A_4250L, 0x0000_0000_0000_0001L, + /* -24 */ 0x69E1_0DE7_6676_D080L, 0x0000_0000_0000_0001L, + /* -23 */ 0x54B4_0B1F_852B_DA00L, 0x0000_0000_0000_0001L, + /* -22 */ 0x43C3_3C19_3756_4800L, 0x0000_0000_0000_0001L, + /* -21 */ 0x6C6B_935B_8BBD_4000L, 0x0000_0000_0000_0001L, + /* -20 */ 0x56BC_75E2_D631_0000L, 0x0000_0000_0000_0001L, + /* -19 */ 0x4563_9182_44F4_0000L, 0x0000_0000_0000_0001L, + /* -18 */ 0x6F05_B59D_3B20_0000L, 0x0000_0000_0000_0001L, + /* -17 */ 0x58D1_5E17_6280_0000L, 0x0000_0000_0000_0001L, + /* -16 */ 0x470D_E4DF_8200_0000L, 0x0000_0000_0000_0001L, + /* -15 */ 0x71AF_D498_D000_0000L, 0x0000_0000_0000_0001L, + /* -14 */ 0x5AF3_107A_4000_0000L, 0x0000_0000_0000_0001L, + /* -13 */ 0x48C2_7395_0000_0000L, 0x0000_0000_0000_0001L, + /* -12 */ 0x746A_5288_0000_0000L, 0x0000_0000_0000_0001L, + /* -11 */ 0x5D21_DBA0_0000_0000L, 0x0000_0000_0000_0001L, + /* -10 */ 0x4A81_7C80_0000_0000L, 0x0000_0000_0000_0001L, + /* -9 */ 0x7735_9400_0000_0000L, 0x0000_0000_0000_0001L, + /* -8 */ 0x5F5E_1000_0000_0000L, 0x0000_0000_0000_0001L, + /* -7 */ 0x4C4B_4000_0000_0000L, 0x0000_0000_0000_0001L, + /* -6 */ 0x7A12_0000_0000_0000L, 0x0000_0000_0000_0001L, + /* -5 */ 0x61A8_0000_0000_0000L, 0x0000_0000_0000_0001L, + /* -4 */ 0x4E20_0000_0000_0000L, 0x0000_0000_0000_0001L, + /* -3 */ 0x7D00_0000_0000_0000L, 0x0000_0000_0000_0001L, + /* -2 */ 0x6400_0000_0000_0000L, 0x0000_0000_0000_0001L, + /* -1 */ 0x5000_0000_0000_0000L, 0x0000_0000_0000_0001L, + /* 0 */ 0x4000_0000_0000_0000L, 0x0000_0000_0000_0001L, + /* 1 */ 0x6666_6666_6666_6666L, 0x3333_3333_3333_3334L, + /* 2 */ 0x51EB_851E_B851_EB85L, 0x0F5C_28F5_C28F_5C29L, + /* 3 */ 0x4189_374B_C6A7_EF9DL, 0x5916_872B_020C_49BBL, + /* 4 */ 0x68DB_8BAC_710C_B295L, 0x74F0_D844_D013_A92BL, + /* 5 */ 0x53E2_D623_8DA3_C211L, 0x43F3_E037_0CDC_8755L, + /* 6 */ 0x431B_DE82_D7B6_34DAL, 0x698F_E692_70B0_6C44L, + /* 7 */ 0x6B5F_CA6A_F2BD_215EL, 0x0F4C_A41D_811A_46D4L, + /* 8 */ 0x55E6_3B88_C230_E77EL, 0x3F70_834A_CDAE_9F10L, + /* 9 */ 0x44B8_2FA0_9B5A_52CBL, 0x4C5A_02A2_3E25_4C0DL, + /* 10 */ 0x6DF3_7F67_5EF6_EADFL, 0x2D5C_D103_96A2_1347L, + /* 11 */ 0x57F5_FF85_E592_557FL, 0x3DE3_DA69_454E_75D3L, + /* 12 */ 0x465E_6604_B7A8_4465L, 0x7E4F_E1ED_D10B_9175L, + /* 13 */ 0x7097_09A1_25DA_0709L, 0x4A19_697C_81AC_1BEFL, + /* 14 */ 0x5A12_6E1A_84AE_6C07L, 0x54E1_2130_67BC_E326L, + /* 15 */ 0x480E_BE7B_9D58_566CL, 0x43E7_4DC0_52FD_8285L, + /* 16 */ 0x734A_CA5F_6226_F0ADL, 0x530B_AF9A_1E62_6A6DL, + /* 17 */ 0x5C3B_D519_1B52_5A24L, 0x426F_BFAE_7EB5_21F1L, + /* 18 */ 0x49C9_7747_490E_AE83L, 0x4EBF_CC8B_9890_E7F4L, + /* 19 */ 0x760F_253E_DB4A_B0D2L, 0x4ACC_7A78_F41B_0CBAL, + /* 20 */ 0x5E72_8432_4908_8D75L, 0x223D_2EC7_29AF_3D62L, + /* 21 */ 0x4B8E_D028_3A6D_3DF7L, 0x34FD_BF05_BAF2_9781L, + /* 22 */ 0x78E4_8040_5D7B_9658L, 0x54C9_31A2_C4B7_58CFL, + /* 23 */ 0x60B6_CD00_4AC9_4513L, 0x5D6D_C14F_03C5_E0A5L, + /* 24 */ 0x4D5F_0A66_A23A_9DA9L, 0x3124_9AA5_9C9E_4D51L, + /* 25 */ 0x7BCB_43D7_69F7_62A8L, 0x4EA0_F76F_60FD_4882L, + /* 26 */ 0x6309_0312_BB2C_4EEDL, 0x254D_92BF_80CA_A068L, + /* 27 */ 0x4F3A_68DB_C8F0_3F24L, 0x1DD7_A899_33D5_4D20L, + /* 28 */ 0x7EC3_DAF9_4180_6506L, 0x62F2_A75B_8622_1500L, + /* 29 */ 0x6569_7BFA_9ACD_1D9FL, 0x025B_B916_04E8_10CDL, + /* 30 */ 0x5121_2FFB_AF0A_7E18L, 0x6849_60DE_6A53_40A4L, + /* 31 */ 0x40E7_5996_25A1_FE7AL, 0x203A_B3E5_21DC_33B6L, + /* 32 */ 0x67D8_8F56_A29C_CA5DL, 0x19F7_863B_6960_52BDL, + /* 33 */ 0x5313_A5DE_E87D_6EB0L, 0x7B2C_6B62_BAB3_7564L, + /* 34 */ 0x4276_1E4B_ED31_255AL, 0x2F56_BC4E_FBC2_C450L, + /* 35 */ 0x6A56_96DF_E1E8_3BC3L, 0x6557_93B1_92D1_3A1AL, + /* 36 */ 0x5512_124C_B4B9_C969L, 0x3779_42F4_7574_2E7BL, + /* 37 */ 0x440E_750A_2A2E_3ABAL, 0x5F94_3590_5DF6_8B96L, + /* 38 */ 0x6CE3_EE76_A9E3_912AL, 0x65B9_EF4D_6324_1289L, + /* 39 */ 0x571C_BEC5_54B6_0DBBL, 0x6AFB_25D7_8283_4207L, + /* 40 */ 0x45B0_989D_DD5E_7163L, 0x08C8_EB12_CECF_6806L, + /* 41 */ 0x6F80_F42F_C897_1BD1L, 0x5ADB_11B7_B14B_D9A3L, + /* 42 */ 0x5933_F68C_A078_E30EL, 0x157C_0E2C_8DD6_47B5L, + /* 43 */ 0x475C_C53D_4D2D_8271L, 0x5DFC_D823_A4AB_6C91L, + /* 44 */ 0x722E_0862_1515_9D82L, 0x632E_269F_6DDF_141BL, + /* 45 */ 0x5B58_06B4_DDAA_E468L, 0x4F58_1EE5_F17F_4349L, + /* 46 */ 0x4913_3890_B155_8386L, 0x72AC_E584_C132_9C3BL, + /* 47 */ 0x74EB_8DB4_4EEF_38D7L, 0x6AAE_3C07_9B84_2D2AL, + /* 48 */ 0x5D89_3E29_D8BF_60ACL, 0x5558_3006_1603_5755L, + /* 49 */ 0x4AD4_31BB_13CC_4D56L, 0x7779_C004_DE69_12ABL, + /* 50 */ 0x77B9_E92B_52E0_7BBEL, 0x258F_99A1_63DB_5111L, + /* 51 */ 0x5FC7_EDBC_424D_2FCBL, 0x37A6_1481_1CAF_740DL, + /* 52 */ 0x4C9F_F163_683D_BFD5L, 0x7951_AA00_E3BF_900BL, + /* 53 */ 0x7A99_8238_A6C9_32EFL, 0x754F_7667_D2CC_19ABL, + /* 54 */ 0x6214_682D_523A_8F26L, 0x2AA5_F853_0F09_AE22L, + /* 55 */ 0x4E76_B9BD_DB62_0C1EL, 0x5551_9375_A5A1_581BL, + /* 56 */ 0x7D8A_C2C9_5F03_4697L, 0x3BB5_B8BC_3C35_59C5L, + /* 57 */ 0x646F_023A_B269_0545L, 0x7C91_6096_9691_149EL, + /* 58 */ 0x5058_CE95_5B87_376BL, 0x16DA_B3AB_ABA7_43B2L, + /* 59 */ 0x4047_0BAA_AF9F_5F88L, 0x78AE_F622_EFB9_02F5L, + /* 60 */ 0x66D8_12AA_B298_98DBL, 0x0DE4_BD04_B2C1_9E54L, + /* 61 */ 0x5246_7555_5BAD_4715L, 0x57EA_30D0_8F01_4B76L, + /* 62 */ 0x41D1_F777_7C8A_9F44L, 0x4654_F3DA_0C01_092CL, + /* 63 */ 0x694F_F258_C744_3207L, 0x23BB_1FC3_4668_0EACL, + /* 64 */ 0x543F_F513_D29C_F4D2L, 0x4FC8_E635_D1EC_D88AL, + /* 65 */ 0x4366_5DA9_754A_5D75L, 0x263A_51C4_A7F0_AD3BL, + /* 66 */ 0x6BD6_FC42_5543_C8BBL, 0x56C3_B607_731A_AEC4L, + /* 67 */ 0x5645_969B_7769_6D62L, 0x789C_919F_8F48_8BD0L, + /* 68 */ 0x4504_787C_5F87_8AB5L, 0x46E3_A7B2_D906_D640L, + /* 69 */ 0x6E6D_8D93_CC0C_1122L, 0x3E39_0C51_5B3E_239AL, + /* 70 */ 0x5857_A476_3CD6_741BL, 0x4B60_D6A7_7C31_B615L, + /* 71 */ 0x46AC_8391_CA45_29AFL, 0x55E7_121F_968E_2B44L, + /* 72 */ 0x7114_05B6_106E_A919L, 0x0971_B698_F0E3_786DL, + /* 73 */ 0x5A76_6AF8_0D25_5414L, 0x078E_2BAD_8D82_C6BDL, + /* 74 */ 0x485E_BBF9_A41D_DCDCL, 0x6C71_BC8A_D79B_D231L, + /* 75 */ 0x73CA_C65C_39C9_6161L, 0x2D82_C744_8C2C_8382L, + /* 76 */ 0x5CA2_3849_C7D4_4DE7L, 0x3E02_3903_A356_CF9BL, + /* 77 */ 0x4A1B_603B_0643_7185L, 0x7E68_2D9C_82AB_D949L, + /* 78 */ 0x7692_3391_A39F_1C09L, 0x4A40_48FA_6AAC_8EDBL, + /* 79 */ 0x5EDB_5C74_82E5_B007L, 0x5500_3A61_EEF0_7249L, + /* 80 */ 0x4BE2_B05D_3584_8CD2L, 0x7733_61E7_F259_F507L, + /* 81 */ 0x796A_B3C8_55A0_E151L, 0x3EB8_9CA6_508F_EE71L, + /* 82 */ 0x6122_296D_114D_810DL, 0x7EFA_16EB_73A6_585BL, + /* 83 */ 0x4DB4_EDF0_DAA4_673EL, 0x3261_ABEF_8FB8_46AFL, + /* 84 */ 0x7C54_AFE7_C43A_3ECAL, 0x1D69_1318_E5F3_A44BL, + /* 85 */ 0x6376_F31F_D02E_98A1L, 0x6454_0F47_1E5C_836FL, + /* 86 */ 0x4F92_5C19_7358_7A1BL, 0x0376_729F_4B7D_35F3L, + /* 87 */ 0x7F50_935B_EBC0_C35EL, 0x38BD_8432_1261_EFEBL, + /* 88 */ 0x65DA_0F7C_BC9A_35E5L, 0x13CA_D028_0EB4_BFEFL, + /* 89 */ 0x517B_3F96_FD48_2B1DL, 0x5CA2_4020_0BC3_CCBFL, + /* 90 */ 0x412F_6612_6439_BC17L, 0x63B5_0019_A303_0A33L, + /* 91 */ 0x684B_D683_D38F_9359L, 0x1F88_0029_04D1_A9EAL, + /* 92 */ 0x536F_DECF_DC72_DC47L, 0x32D3_3354_03DA_EE55L, + /* 93 */ 0x42BF_E573_16C2_49D2L, 0x5BDC_2910_0315_8B77L, + /* 94 */ 0x6ACC_A251_BE03_A951L, 0x12F9_DB4C_D1BC_1258L, + /* 95 */ 0x5570_81DA_FE69_5440L, 0x7594_AF70_A7C9_A847L, + /* 96 */ 0x445A_017B_FEBA_A9CDL, 0x4476_F2C0_863A_ED06L, + /* 97 */ 0x6D5C_CF2C_CAC4_42E2L, 0x3A57_EACD_A391_7B3CL, + /* 98 */ 0x577D_728A_3BD0_3581L, 0x7B79_88A4_82DA_C8FDL, + /* 99 */ 0x45FD_F53B_630C_F79BL, 0x15FA_D3B6_CF15_6D97L, + /* 100 */ 0x6FFC_BB92_3814_BF5EL, 0x565E_1F8A_E4EF_15BEL, + /* 101 */ 0x5996_FC74_F9AA_32B2L, 0x11E4_E608_B725_AAFFL, + /* 102 */ 0x47AB_FD2A_6154_F55BL, 0x27EA_51A0_9284_88CCL, + /* 103 */ 0x72AC_C843_CEEE_555EL, 0x7310_829A_8407_4146L, + /* 104 */ 0x5BBD_6D03_0BF1_DDE5L, 0x4273_9BAE_D005_CDD2L, + /* 105 */ 0x4964_5735_A327_E4B7L, 0x4EC2_E2F2_4004_A4A8L, + /* 106 */ 0x756D_5855_D1D9_6DF2L, 0x4AD1_6B1D_333A_A10CL, + /* 107 */ 0x5DF1_1377_DB14_57F5L, 0x2241_227D_C295_4DA3L, + /* 108 */ 0x4B27_42C6_48DD_132AL, 0x4E9A_81FE_3544_3E1CL, + /* 109 */ 0x783E_D13D_4161_B844L, 0x175D_9CC9_EED3_9694L, + /* 110 */ 0x6032_40FD_CDE7_C69CL, 0x7917_B0A1_8BDC_7876L, + /* 111 */ 0x4CF5_00CB_0B1F_D217L, 0x1412_F3B4_6FE3_9392L, + /* 112 */ 0x7B21_9ADE_7832_E9BEL, 0x5351_85ED_7FD2_85B6L, + /* 113 */ 0x6281_48B1_F9C2_5498L, 0x42A7_9E57_9975_37C5L, + /* 114 */ 0x4ECD_D3C1_949B_76E0L, 0x3552_E512_E12A_9304L, + /* 115 */ 0x7E16_1F9C_20F8_BE33L, 0x6EEB_081E_3510_EB39L, + /* 116 */ 0x64DE_7FB0_1A60_9829L, 0x3F22_6CE4_F740_BC2EL, + /* 117 */ 0x50B1_FFC0_151A_1354L, 0x3281_F0B7_2C33_C9BEL, + /* 118 */ 0x408E_6633_4414_DC43L, 0x4201_8D5F_568F_D498L, + /* 119 */ 0x674A_3D1E_D354_939FL, 0x1CCF_4898_8A7F_BA8DL, + /* 120 */ 0x52A1_CA7F_0F76_DC7FL, 0x30A5_D3AD_3B99_620BL, + /* 121 */ 0x421B_0865_A5F8_B065L, 0x73B7_DC8A_9614_4E6FL, + /* 122 */ 0x69C4_DA3C_3CC1_1A3CL, 0x52BF_C744_2353_B0B1L, + /* 123 */ 0x549D_7B63_63CD_AE96L, 0x7566_3903_4F76_26F4L, + /* 124 */ 0x43B1_2F82_B63E_2545L, 0x4451_C735_D92B_525DL, + /* 125 */ 0x6C4E_B26A_BD30_3BA2L, 0x3A1C_71EF_C1DE_EA2EL, + /* 126 */ 0x56A5_5B88_9759_C94EL, 0x61B0_5B26_34B2_54F2L, + /* 127 */ 0x4551_1606_DF7B_0772L, 0x1AF3_7C1E_908E_AA5BL, + /* 128 */ 0x6EE8_233E_325E_7250L, 0x2B1F_2CFD_B417_76F8L, + /* 129 */ 0x58B9_B5CB_5B7E_C1D9L, 0x6F4C_23FE_29AC_5F2DL, + /* 130 */ 0x46FA_F7D5_E2CB_CE47L, 0x72A3_4FFE_87BD_18F1L, + /* 131 */ 0x7191_8C89_6ADF_B073L, 0x0438_7FFD_A5FB_5B1BL, + /* 132 */ 0x5ADA_D6D4_557F_C05CL, 0x0360_6664_84C9_15AFL, + /* 133 */ 0x48AF_1243_7799_66B0L, 0x02B3_851D_3707_448CL, + /* 134 */ 0x744B_506B_F28F_0AB3L, 0x1DEC_082E_BE72_0746L, + /* 135 */ 0x5D09_0D23_2872_6EF5L, 0x64BC_D358_985B_3905L, + /* 136 */ 0x4A6D_A41C_205B_8BF7L, 0x6A30_A913_AD15_C738L, + /* 137 */ 0x7715_D360_33C5_ACBFL, 0x5D1A_A81F_7B56_0B8CL, + /* 138 */ 0x5F44_A919_C304_8A32L, 0x7DAE_ECE5_FC44_D609L, + /* 139 */ 0x4C36_EDAE_359D_3B5BL, 0x7E25_8A51_969D_7808L, + /* 140 */ 0x79F1_7C49_EF61_F893L, 0x16A2_76E8_F0FB_F33FL, + /* 141 */ 0x618D_FD07_F2B4_C6DCL, 0x121B_9253_F3FC_C299L, + /* 142 */ 0x4E0B_30D3_2890_9F16L, 0x41AF_A843_2997_0214L, + /* 143 */ 0x7CDE_B485_0DB4_31BDL, 0x4F7F_739E_A8F1_9CEDL, + /* 144 */ 0x63E5_5D37_3E29_C164L, 0x3F99_294B_BA5A_E3F1L, + /* 145 */ 0x4FEA_B0F8_FE87_CDE9L, 0x7FAD_BAA2_FB7B_E98DL, + /* 146 */ 0x7FDD_E7F4_CA72_E30FL, 0x7F7C_5DD1_925F_DC15L, + /* 147 */ 0x664B_1FF7_085B_E8D9L, 0x4C63_7E41_41E6_49ABL, + /* 148 */ 0x51D5_B32C_06AF_ED7AL, 0x704F_9834_34B8_3AEFL, + /* 149 */ 0x4177_C289_9EF3_2462L, 0x26A6_135C_F6F9_C8BFL, + /* 150 */ 0x68BF_9DA8_FE51_D3D0L, 0x3DD6_8561_8B29_4132L, + /* 151 */ 0x53CC_7E20_CB74_A973L, 0x4B12_044E_08ED_CDC2L, + /* 152 */ 0x4309_FE80_A2C3_BAC2L, 0x6F41_9D0B_3A57_D7CEL, + /* 153 */ 0x6B43_30CD_D139_2AD1L, 0x3202_94DE_C3BF_BFB0L, + /* 154 */ 0x55CF_5A3E_40FA_88A7L, 0x419B_AA4B_CFCC_995AL, + /* 155 */ 0x44A5_E1CB_672E_D3B9L, 0x1AE2_EEA3_0CA3_ADE1L, + /* 156 */ 0x6DD6_3612_3EB1_52C1L, 0x77D1_7DD1_ADD2_AFCFL, + /* 157 */ 0x57DE_91A8_3227_7567L, 0x7974_64A7_BE42_263FL, + /* 158 */ 0x464B_A7B9_C1B9_2AB9L, 0x4790_5086_31CE_84FFL, + /* 159 */ 0x7079_0C5C_6928_445CL, 0x0C1A_1A70_4FB0_D4CCL, + /* 160 */ 0x59FA_7049_EDB9_D049L, 0x567B_4859_D95A_43D6L, + /* 161 */ 0x47FB_8D07_F161_736EL, 0x11FC_39E1_7AAE_9CABL, + /* 162 */ 0x732C_14D9_8235_857DL, 0x032D_2968_C44A_9445L, + /* 163 */ 0x5C23_43E1_34F7_9DFDL, 0x4F57_5453_D03B_A9D1L, + /* 164 */ 0x49B5_CFE7_5D92_E4CAL, 0x72AC_4376_402F_BB0EL, + /* 165 */ 0x75EF_B30B_C8EB_07ABL, 0x0446_D256_CD19_2B49L, + /* 166 */ 0x5E59_5C09_6D88_D2EFL, 0x1D05_7512_3DAD_BC3AL, + /* 167 */ 0x4B7A_B007_8AD3_DBF2L, 0x4A6A_C40E_97BE_302FL, + /* 168 */ 0x78C4_4CD8_DE1F_C650L, 0x7711_39B0_F2C9_E6B1L, + /* 169 */ 0x609D_0A47_1819_6B73L, 0x78DA_948D_8F07_EBC1L, + /* 170 */ 0x4D4A_6E9F_467A_BC5CL, 0x60AE_DD3E_0C06_5634L, + /* 171 */ 0x7BAA_4A98_70C4_6094L, 0x344A_FB96_79A3_BD20L, + /* 172 */ 0x62EE_A213_8D69_E6DDL, 0x103B_FC78_614F_CA80L, + /* 173 */ 0x4F25_4E76_0ABB_1F17L, 0x2696_6393_810C_A200L, + /* 174 */ 0x7EA2_1723_445E_9825L, 0x2423_D285_9B47_6999L, + /* 175 */ 0x654E_78E9_037E_E01DL, 0x69B6_4204_7C39_2148L, + /* 176 */ 0x510B_93ED_9C65_8017L, 0x6E2B_6803_9694_1AA0L, + /* 177 */ 0x40D6_0FF1_49EA_CCDFL, 0x71BC_5336_1210_154DL, + /* 178 */ 0x67BC_E64E_DCAA_E166L, 0x1C60_8523_5019_BBAEL, + /* 179 */ 0x52FD_850B_E3BB_E784L, 0x7D1A_041C_4014_9625L, + /* 180 */ 0x4264_6A6F_E963_1F9DL, 0x4A7B_367D_0010_781DL, + /* 181 */ 0x6A3A_43E6_4238_3295L, 0x5D91_F0C8_001A_59C8L, + /* 182 */ 0x54FB_6985_01C6_8EDEL, 0x17A7_F3D3_3348_47D4L, + /* 183 */ 0x43FC_546A_67D2_0BE4L, 0x7953_2975_C2A0_3976L, + /* 184 */ 0x6CC6_ED77_0C83_463BL, 0x0EEB_7589_3766_C256L, + /* 185 */ 0x5705_8AC5_A39C_382FL, 0x2589_2AD4_2C52_3512L, + /* 186 */ 0x459E_089E_1C7C_F9BFL, 0x37A0_EF10_2374_F742L, + /* 187 */ 0x6F63_40FC_FA61_8F98L, 0x5901_7E80_38BB_2536L, + /* 188 */ 0x591C_33FD_951A_D946L, 0x7A67_9866_93C8_EA91L, + /* 189 */ 0x4749_C331_4415_7A9FL, 0x151F_AD1E_DCA0_BBA8L, + /* 190 */ 0x720F_9EB5_39BB_F765L, 0x0832_AE97_C767_92A5L, + /* 191 */ 0x5B3F_B22A_9496_5F84L, 0x068E_F213_05EC_7551L, + /* 192 */ 0x48FF_C1BB_AA11_E603L, 0x1ED8_C1A8_D189_F774L, + /* 193 */ 0x74CC_692C_434F_D66BL, 0x4AF4_690E_1C0F_F253L, + /* 194 */ 0x5D70_5423_690C_AB89L, 0x225D_20D8_1673_2843L, + /* 195 */ 0x4AC0_434F_873D_5607L, 0x3517_4D79_AB8F_5369L, + /* 196 */ 0x779A_054C_0B95_5672L, 0x21BE_E25C_45B2_1F0EL, + /* 197 */ 0x5FAE_6AA3_3C77_785BL, 0x3498_B516_9E28_18D8L, + /* 198 */ 0x4C8B_8882_96C5_F9E2L, 0x5D46_F745_4B53_4713L, + /* 199 */ 0x7A78_DA6A_8AD6_5C9DL, 0x7BA4_BED5_4552_0B52L, + /* 200 */ 0x61FA_4855_3BDE_B07EL, 0x2FB6_FF11_0441_A2A8L, + /* 201 */ 0x4E61_D377_6318_8D31L, 0x72F8_CC0D_9D01_4EEDL, + /* 202 */ 0x7D69_5258_9E8D_AEB6L, 0x1E5A_E015_C802_17E1L, + /* 203 */ 0x6454_41E0_7ED7_BEF8L, 0x1848_B344_A001_ACB4L, + /* 204 */ 0x5043_67E6_CBDF_CBF9L, 0x603A_2903_B334_8A2AL, + /* 205 */ 0x4035_ECB8_A319_6FFBL, 0x002E_8736_28F6_D4EEL, + /* 206 */ 0x66BC_ADF4_3828_B32BL, 0x19E4_0B89_DB24_87E3L, + /* 207 */ 0x5230_8B29_C686_F5BCL, 0x14B6_6FA1_7C1D_3983L, + /* 208 */ 0x41C0_6F54_9ED2_5E30L, 0x1091_F2E7_967D_C79CL, + /* 209 */ 0x6933_E554_3150_96B3L, 0x341C_B7D8_F0C9_3F5FL, + /* 210 */ 0x5429_8443_5AA6_DEF5L, 0x767D_5FE0_C0A0_FF80L, + /* 211 */ 0x4354_69CF_7BB8_B25EL, 0x2B97_7FE7_0080_CC66L, + /* 212 */ 0x6BBA_42E5_92C1_1D63L, 0x5F58_CCA4_CD9A_E0A3L, + /* 213 */ 0x562E_9BEA_DBCD_B11CL, 0x4C47_0A1D_7148_B3B6L, + /* 214 */ 0x44F2_1655_7CA4_8DB0L, 0x3D05_A1B1_276D_5C92L, + /* 215 */ 0x6E50_23BB_FAA0_E2B3L, 0x7B3C_35E8_3F15_60E9L, + /* 216 */ 0x5840_1C96_621A_4EF6L, 0x2F63_5E53_65AA_B3EDL, + /* 217 */ 0x4699_B078_4E7B_725EL, 0x591C_4B75_EAEE_F658L, + /* 218 */ 0x70F5_E726_E3F8_B6FDL, 0x74FA_1256_44B1_8A26L, + /* 219 */ 0x5A5E_5285_832D_5F31L, 0x43FB_41DE_9D5A_D4EBL, + /* 220 */ 0x484B_7537_9C24_4C27L, 0x4FFC_34B2_177B_DD89L, + /* 221 */ 0x73AB_EEBF_603A_1372L, 0x4CC6_BAB6_8BF9_6274L, + /* 222 */ 0x5C89_8BCC_4CFB_42C2L, 0x0A38_955E_D661_1B90L, + /* 223 */ 0x4A07_A309_D72F_689BL, 0x21C6_DDE5_784D_AFA7L, + /* 224 */ 0x7672_9E76_2518_A75EL, 0x693E_2FD5_8D49_190BL, + /* 225 */ 0x5EC2_185E_8413_B918L, 0x5431_BFDE_0AA0_E0D5L, + /* 226 */ 0x4BCE_79E5_3676_2DADL, 0x29C1_664B_3BB3_E711L, + /* 227 */ 0x794A_5CA1_F0BD_15E2L, 0x0F9B_D6DE_C5EC_A4E8L, + /* 228 */ 0x6108_4A1B_26FD_AB1BL, 0x2616_457F_04BD_50BAL, + /* 229 */ 0x4DA0_3B48_EBFE_227CL, 0x1E78_3798_D097_73C8L, + /* 230 */ 0x7C33_920E_4663_6A60L, 0x30C0_58F4_80F2_52D9L, + /* 231 */ 0x635C_74D8_384F_884DL, 0x0D66_AD90_6728_4247L, + /* 232 */ 0x4F7D_2A46_9372_D370L, 0x711E_F140_5286_9B6CL, + /* 233 */ 0x7F2E_AA0A_8584_8581L, 0x34FE_4ECD_50D7_5F14L, + /* 234 */ 0x65BE_EE6E_D136_D134L, 0x2A65_0BD7_73DF_7F43L, + /* 235 */ 0x5165_8B8B_DA92_40F6L, 0x551D_A312_C319_329CL, + /* 236 */ 0x411E_093C_AEDB_672BL, 0x5DB1_4F42_35AD_C217L, + /* 237 */ 0x6830_0EC7_7E2B_D845L, 0x7C4E_E536_BC49_368AL, + /* 238 */ 0x5359_A56C_64EF_E037L, 0x7D0B_EA92_303A_9208L, + /* 239 */ 0x42AE_1DF0_50BF_E693L, 0x173C_BBA8_2695_41A0L, + /* 240 */ 0x6AB0_2FE6_E799_70EBL, 0x3EC7_92A6_A422_029AL, + /* 241 */ 0x5559_BFEB_EC7A_C0BCL, 0x3239_421E_E9B4_CEE1L, + /* 242 */ 0x4447_CCBC_BD2F_0096L, 0x5B61_01B2_5490_A581L, + /* 243 */ 0x6D3F_ADFA_C84B_3424L, 0x2BCE_691D_541A_A268L, + /* 244 */ 0x5766_24C8_A03C_29B6L, 0x563E_BA7D_DCE2_1B87L, + /* 245 */ 0x45EB_50A0_8030_215EL, 0x7832_2ECB_171B_4939L, + /* 246 */ 0x6FDE_E767_3380_3564L, 0x59E9_E478_24F8_7527L, + /* 247 */ 0x597F_1F85_C2CC_F783L, 0x6187_E9F9_B72D_2A86L, + /* 248 */ 0x4798_E604_9BD7_2C69L, 0x346C_BB2E_2C24_2205L, + /* 249 */ 0x728E_3CD4_2C8B_7A42L, 0x20AD_F849_E039_D007L, + /* 250 */ 0x5BA4_FD76_8A09_2E9BL, 0x33BE_603B_19C7_D99FL, + /* 251 */ 0x4950_CAC5_3B3A_8BAFL, 0x42FE_B362_7B06_47B3L, + /* 252 */ 0x754E_113B_91F7_45E5L, 0x5197_856A_5E70_72B8L, + /* 253 */ 0x5DD8_0DC9_4192_9E51L, 0x27AC_6ABB_7EC0_5BC6L, + /* 254 */ 0x4B13_3E3A_9ADB_B1DAL, 0x52F0_5562_CBCD_1638L, + /* 255 */ 0x781E_C9F7_5E2C_4FC4L, 0x1E4D_556A_DFAE_89F3L, + /* 256 */ 0x6018_A192_B1BD_0C9CL, 0x7EA4_4455_7FBE_D4C3L, + /* 257 */ 0x4CE0_8142_27CA_707DL, 0x4BB6_9D11_32FF_109CL, + /* 258 */ 0x7B00_CED0_3FAA_4D95L, 0x5F8A_94E8_5198_1A93L, + /* 259 */ 0x6267_0BD9_CC88_3E11L, 0x32D5_43ED_0E13_4875L, + /* 260 */ 0x4EB8_D647_D6D3_64DAL, 0x5BDD_CFF0_D80F_6D2BL, + /* 261 */ 0x7DF4_8A0C_8AEB_D491L, 0x12FC_7FE7_C018_AEABL, + /* 262 */ 0x64C3_A1A3_A256_43A7L, 0x28C9_FFEC_99AD_5889L, + /* 263 */ 0x509C_814F_B511_CFB9L, 0x0707_FFF0_7AF1_13A1L, + /* 264 */ 0x407D_343F_C40E_3FC7L, 0x1F39_998D_2F27_42E7L, + /* 265 */ 0x672E_B9FF_A016_CC71L, 0x7EC2_8F48_4B72_04A4L, + /* 266 */ 0x528B_C7FF_B345_705BL, 0x189B_A5D3_6F8E_6A1DL, + /* 267 */ 0x4209_6CCC_8F6A_C048L, 0x7A16_1E42_BFA5_21B1L, + /* 268 */ 0x69A8_AE14_18AA_CD41L, 0x4356_96D1_32A1_CF81L, + /* 269 */ 0x5486_F1A9_AD55_7101L, 0x1C45_4574_2881_72CEL, + /* 270 */ 0x439F_27BA_F111_2734L, 0x169D_D129_BA01_28A5L, + /* 271 */ 0x6C31_D92B_1B4E_A520L, 0x242F_B50F_9001_DAA1L, + /* 272 */ 0x568E_4755_AF72_1DB3L, 0x368C_90D9_4001_7BB4L, + /* 273 */ 0x453E_9F77_BF8E_7E29L, 0x120A_0D7A_999A_C95DL, + /* 274 */ 0x6ECA_98BF_98E3_FD0EL, 0x5010_1590_F5C4_7561L, + /* 275 */ 0x58A2_13CC_7A4F_FDA5L, 0x2673_4473_F7D0_5DE8L, + /* 276 */ 0x46E8_0FD6_C83F_FE1DL, 0x6B8F_69F6_5FD9_E4B9L, + /* 277 */ 0x7173_4C8A_D9FF_FCFCL, 0x45B2_4323_CC8F_D45CL, + /* 278 */ 0x5AC2_A3A2_47FF_FD96L, 0x6AF5_0283_0A0C_A9E3L, + /* 279 */ 0x489B_B61B_6CCC_CADFL, 0x08C4_0202_6E70_87E9L, + /* 280 */ 0x742C_5692_47AE_1164L, 0x746C_D003_E3E7_3FDBL, + /* 281 */ 0x5CF0_4541_D2F1_A783L, 0x76BD_7336_4FEC_3315L, + /* 282 */ 0x4A59_D101_758E_1F9CL, 0x5EFD_F5C5_0CBC_F5ABL, + /* 283 */ 0x76F6_1B35_88E3_65C7L, 0x4B2F_EFA1_ADFB_22ABL, + /* 284 */ 0x5F2B_48F7_A0B5_EB06L, 0x08F3_261A_F195_B555L, + /* 285 */ 0x4C22_A0C6_1A2B_226BL, 0x20C2_84E2_5ADE_2AABL, + /* 286 */ 0x79D1_013C_F6AB_6A45L, 0x1AD0_D49D_5E30_4444L, + /* 287 */ 0x6174_00FD_9222_BB6AL, 0x48A7_107D_E4F3_69D0L, + /* 288 */ 0x4DF6_6731_41B5_62BBL, 0x53B8_D9FE_50C2_BB0DL, + /* 289 */ 0x7CBD_71E8_6922_3792L, 0x52C1_5CCA_1AD1_2B48L, + /* 290 */ 0x63CA_C186_BA81_C60EL, 0x7567_7D6E_7BDA_8906L, + /* 291 */ 0x4FD5_679E_FB9B_04D8L, 0x5DEC_6458_6315_3A6CL, + /* 292 */ 0x7FBB_D8FE_5F5E_6E27L, 0x497A_3A27_04EE_C3DFL, + }; + +} \ No newline at end of file diff --git a/ujson/templates-js/ToDecimalElem.scala b/ujson/templates-js/ToDecimalElem.scala new file mode 100644 index 000000000..78ca30c57 --- /dev/null +++ b/ujson/templates-js/ToDecimalElem.scala @@ -0,0 +1,24 @@ +package ujson + +object MathUtilsElem{ + def toString(bytes: Array[Elem], index: Int, s: String) = { + val len = s.length + var i = 0 + while (i < len) { + bytes(index + i) = s.charAt(i).toElem + i += 1 + } + len + } +} +object DoubleToDecimalElem{ + def toString(bytes: Array[Elem], index: Int, v: Double) = { + MathUtilsElem.toString(bytes, index, v.toString) + + } +} +object FloatToDecimalElem{ + def toString(bytes: Array[Elem], index: Int, v: Float) = { + MathUtilsElem.toString(bytes, index, v.toString) + } +} \ No newline at end of file diff --git a/ujson/templates-jvm/DoubleToDecimalElem.java b/ujson/templates-jvm/DoubleToDecimalElem.java new file mode 100644 index 000000000..78b4f71d2 --- /dev/null +++ b/ujson/templates-jvm/DoubleToDecimalElem.java @@ -0,0 +1,588 @@ +// DoubleToDecimalfrom +// Jackson +// https://github.com/FasterXML/jackson-core/blob/d26e01fbb4a47354f3c956abe6b120340030aad2/src/main/java/com/fasterxml/jackson/core/io/schubfach/DoubleToDecimal.java +/* + * Copyright 2018-2020 Raffaello Giulietti + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package ujson; + +import static java.lang.Double.doubleToRawLongBits; +import static java.lang.Long.numberOfLeadingZeros; +import static ujson.MathUtils.*; + +/** + * This class exposes a method to render a {@code double} as a string. + * + * @author Raffaello Giulietti + */ +final public class DoubleToDecimalElem { + /* + For full details about this code see the following references: + [1] Giulietti, "The Schubfach way to render doubles", + https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN + [2] IEEE Computer Society, "IEEE Standard for Floating-Point Arithmetic" + [3] Bouvier & Zimmermann, "Division-Free Binary-to-Decimal Conversion" + Divisions are avoided altogether for the benefit of those architectures + that do not provide specific machine instructions or where they are slow. + This is discussed in section 10 of [1]. + */ + + // Sources with the license are here: https://github.com/c4f7fcce9cb06515/Schubfach/blob/3c92d3c9b1fead540616c918cdfef432bca53dfa/todec/src/math/FloatToDecimal.java + + // The precision in bits. + static final int P = 53; + + // Exponent width in bits. + private static final int W = (Double.SIZE - 1) - (P - 1); + + // Minimum value of the exponent: -(2^(W-1)) - P + 3. + static final int Q_MIN = (-1 << W - 1) - P + 3; + + // Maximum value of the exponent: 2^(W-1) - P. + static final int Q_MAX = (1 << W - 1) - P; + + // 10^(E_MIN - 1) <= MIN_VALUE < 10^E_MIN + static final int E_MIN = -323; + + // 10^(E_MAX - 1) <= MAX_VALUE < 10^E_MAX + static final int E_MAX = 309; + + // Threshold to detect tiny values, as in section 8.1.1 of [1] + static final long C_TINY = 3; + + // The minimum and maximum k, as in section 8 of [1] + static final int K_MIN = -324; + static final int K_MAX = 292; + + // H is as in section 8 of [1]. + static final int H = 17; + + // Minimum value of the significand of a normal value: 2^(P-1). + private static final long C_MIN = 1L << P - 1; + + // Mask to extract the biased exponent. + private static final int BQ_MASK = (1 << W) - 1; + + // Mask to extract the fraction bits. + private static final long T_MASK = (1L << P - 1) - 1; + + // Used in rop(). + private static final long MASK_63 = (1L << 63) - 1; + + // Used for left-to-tight digit extraction. + private static final int MASK_28 = (1 << 28) - 1; + + private static final int NON_SPECIAL = 0; + private static final int PLUS_ZERO = 1; + private static final int MINUS_ZERO = 2; + private static final int PLUS_INF = 3; + private static final int MINUS_INF = 4; + private static final int NAN = 5; + + /* + Room for the longer of the forms + -ddddd.dddddddddddd H + 2 characters + -0.00ddddddddddddddddd H + 5 characters + -d.ddddddddddddddddE-eee H + 7 characters + where there are H digits d + */ + public final int MAX_CHARS = H + 7; + + // Numerical results are created here... + private final elem[] bytes; + + // Index into bytes of rightmost valid character. + private int index; + private int offset; + + public DoubleToDecimalElem(elem[] bytes, int offset) { + this.bytes = bytes; + this.offset = offset; + } + + /** + * Returns a string rendering of the {@code double} argument. + * + *

The characters of the result are all drawn from the ASCII set. + *

+ * + *

A decimal is a number of the form + * d×10i + * for some (unique) integers d > 0 and i such that + * d is not a multiple of 10. + * These integers are the significand and + * the exponent, respectively, of the decimal. + * The length of the decimal is the (unique) + * integer n meeting + * 10n-1 ≤ d < 10n. + * + *

The decimal dv + * for a finite positive {@code v} is defined as follows: + *

+ * + *

The (uniquely) selected decimal dv + * is then formatted. + * + *

Let d, i and n be the significand, exponent and + * length of dv, respectively. + * Further, let e = n + i - 1 and let + * d1…dn + * be the usual decimal expansion of the significand. + * Note that d1 ≠ 0 ≠ dn. + *

+ * + * @param v the {@code double} to be rendered. + * @return a string rendering of the argument. + */ + public static int toString(elem[] bytes, int index, double v) { + DoubleToDecimalElem td = new DoubleToDecimalElem(bytes, index); + td.toDecimal(v); + return td.index + 1; + } + + private String toDecimalString(double v) { + switch (toDecimal(v)) { + case NON_SPECIAL: return charsToString(); + case PLUS_ZERO: return "0.0"; + case MINUS_ZERO: return "-0.0"; + case PLUS_INF: return "Infinity"; + case MINUS_INF: return "-Infinity"; + default: return "NaN"; + } + } + + /* + Returns + PLUS_ZERO iff v is 0.0 + MINUS_ZERO iff v is -0.0 + PLUS_INF iff v is POSITIVE_INFINITY + MINUS_INF iff v is NEGATIVE_INFINITY + NAN iff v is NaN + */ + private int toDecimal(double v) { + /* + For full details see references [2] and [1]. + For finite v != 0, determine integers c and q such that + |v| = c 2^q and + Q_MIN <= q <= Q_MAX and + either 2^(P-1) <= c < 2^P (normal) + or 0 < c < 2^(P-1) and q = Q_MIN (subnormal) + */ + long bits = doubleToRawLongBits(v); + long t = bits & T_MASK; + int bq = (int) (bits >>> P - 1) & BQ_MASK; + if (bq < BQ_MASK) { + index = -1; + if (bits < 0) { + append('-'); + } + if (bq != 0) { + // normal value. Here mq = -q + int mq = -Q_MIN + 1 - bq; + long c = C_MIN | t; + // The fast path discussed in section 8.2 of [1]. + if (0 < mq & mq < P) { + long f = c >> mq; + if (f << mq == c) { + return toChars(f, 0); + } + } + return toDecimal(-mq, c, 0); + } + if (t != 0) { + // subnormal value + return t < C_TINY + ? toDecimal(Q_MIN, 10 * t, -1) + : toDecimal(Q_MIN, t, 0); + } + return bits == 0 ? PLUS_ZERO : MINUS_ZERO; + } + if (t != 0) { + return NAN; + } + return bits > 0 ? PLUS_INF : MINUS_INF; + } + + private int toDecimal(int q, long c, int dk) { + /* + The skeleton corresponds to figure 4 of [1]. + The efficient computations are those summarized in figure 7. + Here's a correspondence between Java names and names in [1], + expressed as approximate LaTeX source code and informally. + Other names are identical. + cb: \bar{c} "c-bar" + cbr: \bar{c}_r "c-bar-r" + cbl: \bar{c}_l "c-bar-l" + vb: \bar{v} "v-bar" + vbr: \bar{v}_r "v-bar-r" + vbl: \bar{v}_l "v-bar-l" + rop: r_o' "r-o-prime" + */ + int out = (int) c & 0x1; + long cb = c << 2; + long cbr = cb + 2; + long cbl; + int k; + /* + flog10pow2(e) = floor(log_10(2^e)) + flog10threeQuartersPow2(e) = floor(log_10(3/4 2^e)) + flog2pow10(e) = floor(log_2(10^e)) + */ + if (c != C_MIN | q == Q_MIN) { + // regular spacing + cbl = cb - 2; + k = flog10pow2(q); + } else { + // irregular spacing + cbl = cb - 1; + k = flog10threeQuartersPow2(q); + } + int h = q + flog2pow10(-k) + 2; + + // g1 and g0 are as in section 9.9.3 of [1], so g = g1 2^63 + g0 + long g1 = g1(k); + long g0 = g0(k); + + long vb = rop(g1, g0, cb << h); + long vbl = rop(g1, g0, cbl << h); + long vbr = rop(g1, g0, cbr << h); + + long s = vb >> 2; + if (s >= 100) { + /* + For n = 17, m = 1 the table in section 10 of [1] shows + s' = floor(s / 10) = floor(s 115_292_150_460_684_698 / 2^60) + = floor(s 115_292_150_460_684_698 2^4 / 2^64) + sp10 = 10 s' + tp10 = 10 t' + upin iff u' = sp10 10^k in Rv + wpin iff w' = tp10 10^k in Rv + See section 9.4 of [1]. + */ + long sp10 = 10 * multiplyHigh(s, 115_292_150_460_684_698L << 4); + long tp10 = sp10 + 10; + boolean upin = vbl + out <= sp10 << 2; + boolean wpin = (tp10 << 2) + out <= vbr; + if (upin != wpin) { + return toChars(upin ? sp10 : tp10, k); + } + } + + /* + 10 <= s < 100 or s >= 100 and u', w' not in Rv + uin iff u = s 10^k in Rv + win iff w = t 10^k in Rv + See section 9.4 of [1]. + */ + long t = s + 1; + boolean uin = vbl + out <= s << 2; + boolean win = (t << 2) + out <= vbr; + if (uin != win) { + // Exactly one of u or w lies in Rv. + return toChars(uin ? s : t, k + dk); + } + /* + Both u and w lie in Rv: determine the one closest to v. + See section 9.4 of [1]. + */ + long cmp = vb - (s + t << 1); + return toChars(cmp < 0 || cmp == 0 && (s & 0x1) == 0 ? s : t, k + dk); + } + + /* + Computes rop(cp g 2^(-127)), where g = g1 2^63 + g0 + See section 9.10 and figure 5 of [1]. + */ + private static long rop(long g1, long g0, long cp) { + long x1 = multiplyHigh(g0, cp); + long y0 = g1 * cp; + long y1 = multiplyHigh(g1, cp); + long z = (y0 >>> 1) + x1; + long vbp = y1 + (z >>> 63); + return vbp | (z & MASK_63) + MASK_63 >>> 63; + } + + /* + Formats the decimal f 10^e. + */ + private int toChars(long f, int e) { + /* + For details not discussed here see section 10 of [1]. + Determine len such that + 10^(len-1) <= f < 10^len + */ + int len = flog10pow2(Long.SIZE - numberOfLeadingZeros(f)); + if (f >= pow10(len)) { + len += 1; + } + + /* + Let fp and ep be the original f and e, respectively. + Transform f and e to ensure + 10^(H-1) <= f < 10^H + fp 10^ep = f 10^(e-H) = 0.f 10^e + */ + f *= pow10(H - len); + e += len; + + /* + The toChars?() methods perform left-to-right digits extraction + using ints, provided that the arguments are limited to 8 digits. + Therefore, split the H = 17 digits of f into: + h = the most significant digit of f + m = the next 8 most significant digits of f + l = the last 8, least significant digits of f + For n = 17, m = 8 the table in section 10 of [1] shows + floor(f / 10^8) = floor(193_428_131_138_340_668 f / 2^84) = + floor(floor(193_428_131_138_340_668 f / 2^64) / 2^20) + and for n = 9, m = 8 + floor(hm / 10^8) = floor(1_441_151_881 hm / 2^57) + */ + long hm = multiplyHigh(f, 193_428_131_138_340_668L) >>> 20; + int l = (int) (f - 100_000_000L * hm); + int h = (int) (hm * 1_441_151_881L >>> 57); + int m = (int) (hm - 100_000_000 * h); + + if (0 < e && e <= 7) { + return toChars1(h, m, l, e); + } + if (-3 < e && e <= 0) { + return toChars2(h, m, l, e); + } + return toChars3(h, m, l, e); + } + + private int toChars1(int h, int m, int l, int e) { + /* + 0 < e <= 7: plain format without leading zeroes. + Left-to-right digits extraction: + algorithm 1 in [3], with b = 10, k = 8, n = 28. + */ + appendDigit(h); + int y = y(m); + int t; + int i = 1; + for (; i < e; ++i) { + t = 10 * y; + appendDigit(t >>> 28); + y = t & MASK_28; + } + append('.'); + for (; i <= 8; ++i) { + t = 10 * y; + appendDigit(t >>> 28); + y = t & MASK_28; + } + lowDigits(l); + return NON_SPECIAL; + } + + private int toChars2(int h, int m, int l, int e) { + // -3 < e <= 0: plain format with leading zeroes. + appendDigit(0); + append('.'); + for (; e < 0; ++e) { + appendDigit(0); + } + appendDigit(h); + append8Digits(m); + lowDigits(l); + return NON_SPECIAL; + } + + private int toChars3(int h, int m, int l, int e) { + // -3 >= e | e > 7: computerized scientific notation + appendDigit(h); + append('.'); + append8Digits(m); + lowDigits(l); + exponent(e - 1); + return NON_SPECIAL; + } + + private void lowDigits(int l) { + if (l != 0) { + append8Digits(l); + } + removeTrailingZeroes(); + } + + private void append8Digits(int m) { + /* + Left-to-right digits extraction: + algorithm 1 in [3], with b = 10, k = 8, n = 28. + */ + int y = y(m); + for (int i = 0; i < 8; ++i) { + int t = 10 * y; + appendDigit(t >>> 28); + y = t & MASK_28; + } + } + + private void removeTrailingZeroes() { + while (bytes[offset + index] == '0') { + --index; + } + // ... but do not remove the one directly to the right of '.' + if (bytes[offset + index] == '.') { + ++index; + } + } + + private int y(int a) { + /* + Algorithm 1 in [3] needs computation of + floor((a + 1) 2^n / b^k) - 1 + with a < 10^8, b = 10, k = 8, n = 28. + Noting that + (a + 1) 2^n <= 10^8 2^28 < 10^17 + For n = 17, m = 8 the table in section 10 of [1] leads to: + */ + return (int) (multiplyHigh( + (long) (a + 1) << 28, + 193_428_131_138_340_668L) >>> 20) - 1; + } + + private void exponent(int e) { + append('E'); + if (e < 0) { + append('-'); + e = -e; + } + if (e < 10) { + appendDigit(e); + return; + } + int d; + if (e >= 100) { + /* + For n = 3, m = 2 the table in section 10 of [1] shows + floor(e / 100) = floor(1_311 e / 2^17) + */ + d = e * 1_311 >>> 17; + appendDigit(d); + e -= 100 * d; + } + /* + For n = 2, m = 1 the table in section 10 of [1] shows + floor(e / 10) = floor(103 e / 2^10) + */ + d = e * 103 >>> 10; + appendDigit(d); + appendDigit(e - 10 * d); + } + + private void append(int c) { + bytes[offset + ++index] = (elem) c; + } + + private void appendDigit(int d) { + bytes[offset + ++index] = (elem) ('0' + d); + } + + // Using the deprecated constructor enhances performance. + @SuppressWarnings("deprecation") + private String charsToString() { + return new String(bytes, 0, index + 1); + } + +} \ No newline at end of file diff --git a/ujson/templates-jvm/FloatToDecimalElem.java b/ujson/templates-jvm/FloatToDecimalElem.java new file mode 100644 index 000000000..3ecc6bd72 --- /dev/null +++ b/ujson/templates-jvm/FloatToDecimalElem.java @@ -0,0 +1,560 @@ +// Vendored from Jackson +// https://github.com/FasterXML/jackson-core/blob/d26e01fbb4a47354f3c956abe6b120340030aad2/src/main/java/com/fasterxml/jackson/core/io/schubfach/FloatToDecimal.java +/* + * Copyright 2018-2020 Raffaello Giulietti + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package ujson; + +import static java.lang.Float.floatToRawIntBits; +import static java.lang.Integer.numberOfLeadingZeros; +import static ujson.MathUtils.*; + +/** + * This class exposes a method to render a {@code float} as a string. + * + * @author Raffaello Giulietti + */ +final public class FloatToDecimalElem { + /* + For full details about this code see the following references: + [1] Giulietti, "The Schubfach way to render doubles", + https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN + [2] IEEE Computer Society, "IEEE Standard for Floating-Point Arithmetic" + [3] Bouvier & Zimmermann, "Division-Free Binary-to-Decimal Conversion" + Divisions are avoided altogether for the benefit of those architectures + that do not provide specific machine instructions or where they are slow. + This is discussed in section 10 of [1]. + */ + + // Sources with the license are here: https://github.com/c4f7fcce9cb06515/Schubfach/blob/3c92d3c9b1fead540616c918cdfef432bca53dfa/todec/src/math/FloatToDecimal.java + + // The precision in bits. + static final int P = 24; + + // Exponent width in bits. + private static final int W = (Float.SIZE - 1) - (P - 1); + + // Minimum value of the exponent: -(2^(W-1)) - P + 3. + static final int Q_MIN = (-1 << W - 1) - P + 3; + + // Maximum value of the exponent: 2^(W-1) - P. + static final int Q_MAX = (1 << W - 1) - P; + + // 10^(E_MIN - 1) <= MIN_VALUE < 10^E_MIN + static final int E_MIN = -44; + + // 10^(E_MAX - 1) <= MAX_VALUE < 10^E_MAX + static final int E_MAX = 39; + + // Threshold to detect tiny values, as in section 8.1.1 of [1] + static final int C_TINY = 8; + + // The minimum and maximum k, as in section 8 of [1] + static final int K_MIN = -45; + static final int K_MAX = 31; + + // H is as in section 8 of [1]. + static final int H = 9; + + // Minimum value of the significand of a normal value: 2^(P-1). + private static final int C_MIN = 1 << P - 1; + + // Mask to extract the biased exponent. + private static final int BQ_MASK = (1 << W) - 1; + + // Mask to extract the fraction bits. + private static final int T_MASK = (1 << P - 1) - 1; + + // Used in rop(). + private static final long MASK_32 = (1L << 32) - 1; + + // Used for left-to-tight digit extraction. + private static final int MASK_28 = (1 << 28) - 1; + + private static final int NON_SPECIAL = 0; + private static final int PLUS_ZERO = 1; + private static final int MINUS_ZERO = 2; + private static final int PLUS_INF = 3; + private static final int MINUS_INF = 4; + private static final int NAN = 5; + + /* + Room for the longer of the forms + -ddddd.dddd H + 2 characters + -0.00ddddddddd H + 5 characters + -d.ddddddddE-ee H + 6 characters + where there are H digits d + */ + public final int MAX_CHARS = H + 6; + + // Numerical results are created here... + private final elem[] bytes; + + // Index into buf of rightmost valid character. + private int index; + + private int offset; + public FloatToDecimalElem(elem[] bytes, int offset) { + this.bytes = bytes; + this.offset = offset; + } + + /** + * Returns a string rendering of the {@code float} argument. + * + *

The characters of the result are all drawn from the ASCII set. + *

+ * + *

A decimal is a number of the form + * d×10i + * for some (unique) integers d > 0 and i such that + * d is not a multiple of 10. + * These integers are the significand and + * the exponent, respectively, of the decimal. + * The length of the decimal is the (unique) + * integer n meeting + * 10n-1 ≤ d < 10n. + * + *

The decimal dv + * for a finite positive {@code v} is defined as follows: + *

+ * + *

The (uniquely) selected decimal dv + * is then formatted. + * + *

Let d, i and n be the significand, exponent and + * length of dv, respectively. + * Further, let e = n + i - 1 and let + * d1…dn + * be the usual decimal expansion of the significand. + * Note that d1 ≠ 0 ≠ dn. + *

+ * + * @param v the {@code float} to be rendered. + * @return a string rendering of the argument. + */ + public static int toString(elem[] bytes, int offset, float v) { + FloatToDecimalElem td = new FloatToDecimalElem(bytes, offset); + td.toDecimal(v); + return td.index + 1; + } + + private String toDecimalString(float v) { + switch (toDecimal(v)) { + case NON_SPECIAL: return charsToString(); + case PLUS_ZERO: return "0.0"; + case MINUS_ZERO: return "-0.0"; + case PLUS_INF: return "Infinity"; + case MINUS_INF: return "-Infinity"; + default: return "NaN"; + } + } + + /* + Returns + PLUS_ZERO iff v is 0.0 + MINUS_ZERO iff v is -0.0 + PLUS_INF iff v is POSITIVE_INFINITY + MINUS_INF iff v is NEGATIVE_INFINITY + NAN iff v is NaN + */ + private int toDecimal(float v) { + /* + For full details see references [2] and [1]. + For finite v != 0, determine integers c and q such that + |v| = c 2^q and + Q_MIN <= q <= Q_MAX and + either 2^(P-1) <= c < 2^P (normal) + or 0 < c < 2^(P-1) and q = Q_MIN (subnormal) + */ + int bits = floatToRawIntBits(v); + int t = bits & T_MASK; + int bq = (bits >>> P - 1) & BQ_MASK; + if (bq < BQ_MASK) { + index = -1; + if (bits < 0) { + append('-'); + } + if (bq != 0) { + // normal value. Here mq = -q + int mq = -Q_MIN + 1 - bq; + int c = C_MIN | t; + // The fast path discussed in section 8.2 of [1]. + if (0 < mq & mq < P) { + int f = c >> mq; + if (f << mq == c) { + return toChars(f, 0); + } + } + return toDecimal(-mq, c, 0); + } + if (t != 0) { + // subnormal value + return t < C_TINY + ? toDecimal(Q_MIN, 10 * t, -1) + : toDecimal(Q_MIN, t, 0); + } + return bits == 0 ? PLUS_ZERO : MINUS_ZERO; + } + if (t != 0) { + return NAN; + } + return bits > 0 ? PLUS_INF : MINUS_INF; + } + + private int toDecimal(int q, int c, int dk) { + /* + The skeleton corresponds to figure 4 of [1]. + The efficient computations are those summarized in figure 7. + Also check the appendix. + Here's a correspondence between Java names and names in [1], + expressed as approximate LaTeX source code and informally. + Other names are identical. + cb: \bar{c} "c-bar" + cbr: \bar{c}_r "c-bar-r" + cbl: \bar{c}_l "c-bar-l" + vb: \bar{v} "v-bar" + vbr: \bar{v}_r "v-bar-r" + vbl: \bar{v}_l "v-bar-l" + rop: r_o' "r-o-prime" + */ + int out = c & 0x1; + long cb = c << 2; + long cbr = cb + 2; + long cbl; + int k; + /* + flog10pow2(e) = floor(log_10(2^e)) + flog10threeQuartersPow2(e) = floor(log_10(3/4 2^e)) + flog2pow10(e) = floor(log_2(10^e)) + */ + if (c != C_MIN | q == Q_MIN) { + // regular spacing + cbl = cb - 2; + k = flog10pow2(q); + } else { + // irregular spacing0 + cbl = cb - 1; + k = flog10threeQuartersPow2(q); + } + int h = q + flog2pow10(-k) + 33; + + // g is as in the appendix + long g = g1(k) + 1; + + int vb = rop(g, cb << h); + int vbl = rop(g, cbl << h); + int vbr = rop(g, cbr << h); + + int s = vb >> 2; + if (s >= 100) { + /* + For n = 9, m = 1 the table in section 10 of [1] shows + s' = floor(s / 10) = floor(s 1_717_986_919 / 2^34) + sp10 = 10 s' + tp10 = 10 t' + upin iff u' = sp10 10^k in Rv + wpin iff w' = tp10 10^k in Rv + See section 9.4 of [1]. + */ + int sp10 = 10 * (int) (s * 1_717_986_919L >>> 34); + int tp10 = sp10 + 10; + boolean upin = vbl + out <= sp10 << 2; + boolean wpin = (tp10 << 2) + out <= vbr; + if (upin != wpin) { + return toChars(upin ? sp10 : tp10, k); + } + } + + /* + 10 <= s < 100 or s >= 100 and u', w' not in Rv + uin iff u = s 10^k in Rv + win iff w = t 10^k in Rv + See section 9.4 of [1]. + */ + int t = s + 1; + boolean uin = vbl + out <= s << 2; + boolean win = (t << 2) + out <= vbr; + if (uin != win) { + // Exactly one of u or w lies in Rv. + return toChars(uin ? s : t, k + dk); + } + /* + Both u and w lie in Rv: determine the one closest to v. + See section 9.4 of [1]. + */ + int cmp = vb - (s + t << 1); + return toChars(cmp < 0 || cmp == 0 && (s & 0x1) == 0 ? s : t, k + dk); + } + + /* + Computes rop(cp g 2^(-95)) + See appendix and figure 8 of [1]. + */ + private static int rop(long g, long cp) { + long x1 = multiplyHigh(g, cp); + long vbp = x1 >>> 31; + return (int) (vbp | (x1 & MASK_32) + MASK_32 >>> 32); + } + + /* + Formats the decimal f 10^e. + */ + private int toChars(int f, int e) { + /* + For details not discussed here see section 10 of [1]. + Determine len such that + 10^(len-1) <= f < 10^len + */ + int len = flog10pow2(Integer.SIZE - numberOfLeadingZeros(f)); + if (f >= pow10(len)) { + len += 1; + } + + /* + Let fp and ep be the original f and e, respectively. + Transform f and e to ensure + 10^(H-1) <= f < 10^H + fp 10^ep = f 10^(e-H) = 0.f 10^e + */ + f *= pow10(H - len); + e += len; + + /* + The toChars?() methods perform left-to-right digits extraction + using ints, provided that the arguments are limited to 8 digits. + Therefore, split the H = 9 digits of f into: + h = the most significant digit of f + l = the last 8, least significant digits of f + For n = 9, m = 8 the table in section 10 of [1] shows + floor(f / 10^8) = floor(1_441_151_881 f / 2^57) + */ + int h = (int) (f * 1_441_151_881L >>> 57); + int l = f - 100_000_000 * h; + + if (0 < e && e <= 7) { + return toChars1(h, l, e); + } + if (-3 < e && e <= 0) { + return toChars2(h, l, e); + } + return toChars3(h, l, e); + } + + private int toChars1(int h, int l, int e) { + /* + 0 < e <= 7: plain format without leading zeroes. + Left-to-right digits extraction: + algorithm 1 in [3], with b = 10, k = 8, n = 28. + */ + appendDigit(h); + int y = y(l); + int t; + int i = 1; + for (; i < e; ++i) { + t = 10 * y; + appendDigit(t >>> 28); + y = t & MASK_28; + } + append('.'); + for (; i <= 8; ++i) { + t = 10 * y; + appendDigit(t >>> 28); + y = t & MASK_28; + } + removeTrailingZeroes(); + return NON_SPECIAL; + } + + private int toChars2(int h, int l, int e) { + // -3 < e <= 0: plain format with leading zeroes. + appendDigit(0); + append('.'); + for (; e < 0; ++e) { + appendDigit(0); + } + appendDigit(h); + append8Digits(l); + removeTrailingZeroes(); + return NON_SPECIAL; + } + + private int toChars3(int h, int l, int e) { + // -3 >= e | e > 7: computerized scientific notation + appendDigit(h); + append('.'); + append8Digits(l); + removeTrailingZeroes(); + exponent(e - 1); + return NON_SPECIAL; + } + + private void append8Digits(int m) { + /* + Left-to-right digits extraction: + algorithm 1 in [3], with b = 10, k = 8, n = 28. + */ + int y = y(m); + for (int i = 0; i < 8; ++i) { + int t = 10 * y; + appendDigit(t >>> 28); + y = t & MASK_28; + } + } + + private void removeTrailingZeroes() { + while (bytes[offset + index] == '0') { + --index; + } + // ... but do not remove the one directly to the right of '.' + if (bytes[offset + index] == '.') { + ++index; + } + } + + private int y(int a) { + /* + Algorithm 1 in [3] needs computation of + floor((a + 1) 2^n / b^k) - 1 + with a < 10^8, b = 10, k = 8, n = 28. + Noting that + (a + 1) 2^n <= 10^8 2^28 < 10^17 + For n = 17, m = 8 the table in section 10 of [1] leads to: + */ + return (int) (multiplyHigh( + (long) (a + 1) << 28, + 193_428_131_138_340_668L) >>> 20) - 1; + } + + private void exponent(int e) { + append('E'); + if (e < 0) { + append('-'); + e = -e; + } + if (e < 10) { + appendDigit(e); + return; + } + /* + For n = 2, m = 1 the table in section 10 of [1] shows + floor(e / 10) = floor(103 e / 2^10) + */ + int d = e * 103 >>> 10; + appendDigit(d); + appendDigit(e - 10 * d); + } + + private void append(int c) { + bytes[offset + ++index] = (elem) c; + } + + private void appendDigit(int d) { + bytes[offset + ++index] = (elem) ('0' + d); + } + + // Using the deprecated constructor enhances performance. + @SuppressWarnings("deprecation") + private String charsToString() { + return new String(bytes, 0, index + 1); + } + +} \ No newline at end of file diff --git a/ujson/templates-native/ToDecimalElem.scala b/ujson/templates-native/ToDecimalElem.scala new file mode 100644 index 000000000..78ca30c57 --- /dev/null +++ b/ujson/templates-native/ToDecimalElem.scala @@ -0,0 +1,24 @@ +package ujson + +object MathUtilsElem{ + def toString(bytes: Array[Elem], index: Int, s: String) = { + val len = s.length + var i = 0 + while (i < len) { + bytes(index + i) = s.charAt(i).toElem + i += 1 + } + len + } +} +object DoubleToDecimalElem{ + def toString(bytes: Array[Elem], index: Int, v: Double) = { + MathUtilsElem.toString(bytes, index, v.toString) + + } +} +object FloatToDecimalElem{ + def toString(bytes: Array[Elem], index: Int, v: Float) = { + MathUtilsElem.toString(bytes, index, v.toString) + } +} \ No newline at end of file diff --git a/ujson/templates/BaseElemRenderer.scala b/ujson/templates/BaseElemRenderer.scala index 048261109..968093a83 100644 --- a/ujson/templates/BaseElemRenderer.scala +++ b/ujson/templates/BaseElemRenderer.scala @@ -1,6 +1,6 @@ package ujson import scala.annotation.switch -import upickle.core.{ArrVisitor, ObjVisitor} +import upickle.core.{ArrVisitor, ObjVisitor, ElemBuilder, RenderUtils} /** * A specialized JSON renderer that can render Elems (Chars or Bytes) directly @@ -95,47 +95,28 @@ class BaseElemRenderer[T <: upickle.core.ElemOps.Output] def visitNull(index: Int) = { flushBuffer() - elemBuilder.ensureLength(4) - elemBuilder.appendUnsafe('n') - elemBuilder.appendUnsafe('u') - elemBuilder.appendUnsafe('l') - elemBuilder.appendUnsafe('l') + BaseElemRenderer.appendNull(elemBuilder) flushElemBuilder() out } def visitFalse(index: Int) = { flushBuffer() - elemBuilder.ensureLength(5) - elemBuilder.appendUnsafe('f') - elemBuilder.appendUnsafe('a') - elemBuilder.appendUnsafe('l') - elemBuilder.appendUnsafe('s') - elemBuilder.appendUnsafe('e') + BaseElemRenderer.appendFalse(elemBuilder) flushElemBuilder() out } def visitTrue(index: Int) = { flushBuffer() - elemBuilder.ensureLength(4) - elemBuilder.appendUnsafe('t') - elemBuilder.appendUnsafe('r') - elemBuilder.appendUnsafe('u') - elemBuilder.appendUnsafe('e') + BaseElemRenderer.appendTrue(elemBuilder) flushElemBuilder() out } def visitFloat64StringParts(s: CharSequence, decIndex: Int, expIndex: Int, index: Int) = { flushBuffer() - elemBuilder.ensureLength(s.length()) - var i = 0 - val sLength = s.length - while(i < sLength){ - elemBuilder.appendUnsafeC(s.charAt(i)) - i += 1 - } + BaseElemRenderer.appendKnownAsciiString(elemBuilder, s) flushElemBuilder() out } @@ -146,12 +127,21 @@ class BaseElemRenderer[T <: upickle.core.ElemOps.Output] case Float.NegativeInfinity => visitNonNullString("-Infinity", -1) case d if java.lang.Float.isNaN(d) => visitNonNullString("NaN", -1) case d => + // Ensure that for whole numbers that can be exactly represented by an + // int or long, write them in int notation with decimal points or exponents val i = d.toInt - if (d == i) visitFloat64StringParts(i.toString, -1, -1, index) - else super.visitFloat32(d, index) - flushBuffer() + if (d == i) visitInt32(i, index) + else { + val i = d.toLong + flushBuffer() + if (i == d) BaseElemRenderer.appendKnownAsciiString(elemBuilder, d.toString) + else { + elemBuilder.ensureLength(15) + elemBuilder.length += ujson.FloatToDecimalElem.toString(elemBuilder.arr, elemBuilder.length, d) + } + flushElemBuilder() + } } - flushElemBuilder() out } @@ -161,15 +151,50 @@ class BaseElemRenderer[T <: upickle.core.ElemOps.Output] case Double.NegativeInfinity => visitNonNullString("-Infinity", -1) case d if java.lang.Double.isNaN(d) => visitNonNullString("NaN", -1) case d => + // Ensure that for whole numbers that can be exactly represented by an + // int or long, write them in int notation with decimal points or exponents val i = d.toInt - if (d == i) visitFloat64StringParts(i.toString, -1, -1, index) - else super.visitFloat64(d, index) - flushBuffer() + if (d == i) visitInt32(i, index) + else { + val i = d.toLong + flushBuffer() + if (i == d) BaseElemRenderer.appendKnownAsciiString(elemBuilder, i.toString) + else { + elemBuilder.ensureLength(24) + elemBuilder.length += ujson.DoubleToDecimalElem.toString(elemBuilder.arr, elemBuilder.length, d) + } + flushElemBuilder() + } } + out + } + + override def visitInt32(i: Int, index: Int) = { + flushBuffer() + BaseElemRenderer.appendIntString(elemBuilder, i) + flushElemBuilder() + out + } + + override def visitInt64(i: Long, index: Int) = { + flushBuffer() + if (math.abs(i) > 9007199254740992L /*math.pow(2, 53)*/ || + i == -9223372036854775808L /*Long.MinValue*/ ) { + elemBuilder.append('"') + BaseElemRenderer.appendLongString(elemBuilder, i) + elemBuilder.append('"') + } else BaseElemRenderer.appendLongString(elemBuilder, i) flushElemBuilder() out } + override def visitUInt64(i: Long, index: Int) = { + val int = i.toInt + if (int == i) visitInt32(int, index) + else super.visitUInt64(i, index) + out + } + def visitString(s: CharSequence, index: Int) = { if (s eq null) visitNull(index) @@ -192,11 +217,199 @@ class BaseElemRenderer[T <: upickle.core.ElemOps.Output] else { var i = indent * depth elemBuilder.ensureLength(i + 1) - elemBuilder.appendUnsafe('\n') - while(i > 0) { - elemBuilder.appendUnsafe(' ') - i -= 1 - } + BaseElemRenderer.renderIdent(elemBuilder.arr, elemBuilder.length, i) + elemBuilder.length += i + 1 } } } + +object BaseElemRenderer{ + private def renderIdent(arr: Array[Elem], length: Int, i0: Int) = { + var i = i0 + arr(length) = '\n' + while (i > 0) { + arr(length + i) = ' ' + i -= 1 + } + } + + private def appendIntString(eb: ElemBuilder, i0: Int) = { + val size = RenderUtils.intStringSize(i0) + val newLength = eb.length + size + eb.ensureLength(size) + appendIntString0(i0, newLength, eb.arr) + eb.length = newLength + } + + private def appendIntString0(i0: Int, index: Int, arr: Array[Elem]) = { + // Copied from java.lang.Integer.getChars + var i = i0 + var q = 0 + var r = 0 + var charPos = index + val negative = i < 0 + if (!negative) i = -i + // Generate two digits per iteration + while (i <= -100) { + q = i / 100 + r = (q * 100) - i + i = q + charPos -= 1 + arr(charPos) = DigitOnes(r) + charPos -= 1 + arr(charPos) = DigitTens(r) + } + // We know there are at most two digits left at this point. + q = i / 10 + r = (q * 10) - i + charPos -= 1 + arr(charPos) = ('0' + r).toElem + // Whatever left is the remaining digit. + if (q < 0) { + charPos -= 1 + arr(charPos) = ('0' - q).toElem + } + if (negative) { + charPos -= 1; + arr(charPos) = '-'.toElem + } + charPos + } + + private def appendLongString(eb: ElemBuilder, i0: Long) = { + val size = RenderUtils.longStringSize(i0) + val newLength = eb.length + size + eb.ensureLength(size) + appendLongString0(i0, newLength, eb.arr) + eb.length = newLength + } + + private def appendLongString0(i0: Long, index: Int, buf: Array[Elem]) = { + // Copied from java.lang.Long.getChars + var i = i0 + var q = 0L + var r = 0 + var charPos = index + val negative = i < 0 + if (!negative) i = -i + // Get 2 digits/iteration using longs until quotient fits into an int + while (i <= Integer.MIN_VALUE) { + q = i / 100 + r = ((q * 100) - i).toInt + i = q + charPos -= 1 + buf(charPos) = DigitOnes(r) + charPos -= 1 + buf(charPos) = DigitTens(r) + } + // Get 2 digits/iteration using ints + var q2 = 0 + var i2 = i.toInt + while (i2 <= -100) { + q2 = i2 / 100 + r = (q2 * 100) - i2 + i2 = q2 + charPos -= 1; + buf(charPos) = DigitOnes(r) + charPos -= 1; + buf(charPos) = DigitTens(r) + } + // We know there are at most two digits left at this point. + q2 = i2 / 10 + r = (q2 * 10) - i2 + charPos -= 1 + buf(charPos) = ('0' + r).toElem + // Whatever left is the remaining digit. + if (q2 < 0) { + charPos -= 1 + buf(charPos) = ('0' - q2).toElem + } + if (negative) { + charPos -= 1 + buf(charPos) = '-'.toElem + } + charPos + } + + private val DigitTens = Array[Elem]( + '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', + '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', + '2', '2', '2', '2', '2', '2', '2', '2', '2', '2', + '3', '3', '3', '3', '3', '3', '3', '3', '3', '3', + '4', '4', '4', '4', '4', '4', '4', '4', '4', '4', + '5', '5', '5', '5', '5', '5', '5', '5', '5', '5', + '6', '6', '6', '6', '6', '6', '6', '6', '6', '6', + '7', '7', '7', '7', '7', '7', '7', '7', '7', '7', + '8', '8', '8', '8', '8', '8', '8', '8', '8', '8', + '9', '9', '9', '9', '9', '9', '9', '9', '9', '9', + ) + + private val DigitOnes = Array[Elem]( + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + ) + + private def appendNull(eb: ElemBuilder) = { + eb.ensureLength(4) + appendNull0(eb.arr, eb.length) + eb.length += 4 + } + + private def appendNull0(arr: Array[Elem], arrOffset: Int) = { + arr(arrOffset) = 'n'.toElem + arr(arrOffset + 1) = 'u'.toElem + arr(arrOffset + 2) = 'l'.toElem + arr(arrOffset + 3) = 'l'.toElem + } + + private def appendTrue(eb: ElemBuilder) = { + eb.ensureLength(4) + appendTrue0(eb.arr, eb.length) + eb.length += 4 + } + + private def appendTrue0(arr: Array[Elem], arrOffset: Int) = { + arr(arrOffset) = 't'.toElem + arr(arrOffset + 1) = 'r'.toElem + arr(arrOffset + 2) = 'u'.toElem + arr(arrOffset + 3) = 'e'.toElem + } + + private def appendFalse(eb: ElemBuilder) = { + eb.ensureLength(5) + appendFalse0(eb.arr, eb.length) + eb.length += 5 + } + + private def appendFalse0(arr: Array[Elem], arrOffset: Int) = { + arr(arrOffset) = 'f'.toElem + arr(arrOffset + 1) = 'a'.toElem + arr(arrOffset + 2) = 'l'.toElem + arr(arrOffset + 3) = 's'.toElem + arr(arrOffset + 4) = 'e'.toElem + } + + private def appendKnownAsciiString(eb: ElemBuilder, s: CharSequence) = { + val sLength = s.length + eb.ensureLength(sLength) + appendKnownAsciiString0(eb.arr, eb.length, s, sLength) + + eb.length += sLength + } + + private def appendKnownAsciiString0(arr: Array[Elem], arrOffset: Int, s: CharSequence, sLength: Int) = { + var i = 0 + while (i < sLength) { + arr(arrOffset + i) = s.charAt(i).toElem + i += 1 + } + } +} \ No newline at end of file diff --git a/upickle/test/src/upickle/AdvancedTests.scala b/upickle/test/src/upickle/AdvancedTests.scala index d4985583f..2fe53033c 100644 --- a/upickle/test/src/upickle/AdvancedTests.scala +++ b/upickle/test/src/upickle/AdvancedTests.scala @@ -340,44 +340,6 @@ object AdvancedTests extends TestSuite { val result = upickle.default.read[Node371](input) assert(result == expected) } - - test("hash-collision") { - def zeroHashCodeStrings: Iterator[String] = { - def charAndHash(h: Int): Iterator[(Char, Int)] = ('!' to '~').iterator.map(ch => (ch, (h + ch) * 31)) - - for { - (ch0, h0) <- charAndHash(0) - (ch1, h1) <- charAndHash(h0) - (ch2, h2) <- charAndHash(h1) if ((h2 + 32) * 923521 ^ (h2 + 127) * 923521) < 0 - (ch3, h3) <- charAndHash(h2) if ((h3 + 32) * 29791 ^ (h3 + 127) * 29791) < 0 - (ch4, h4) <- charAndHash(h3) if ((h4 + 32) * 961 ^ (h4 + 127) * 961) < 0 - (ch5, h5) <- charAndHash(h4) if ((h5 + 32) * 31 ^ (h5 + 127) * 31) < 0 - (ch6, h6) <- charAndHash(h5) if (h6 + 32 ^ h6 + 127) < 0 - (ch7, _) <- charAndHash(h6) if h6 + ch7 == 0 - } yield new String(Array(ch0, ch1, ch2, ch3, ch4, ch5, ch6, ch7)) - } - - val jsonString = - zeroHashCodeStrings - .map(s => ujson.write(s)) - .take(1000000) - .mkString("{", s":null,", ":null}") - - test("issue-416") { - withTimeout { upickle.default.read[Foo416](jsonString) } - } - test("issue-446") { - sys.props("java.vm.name") match { - case "Scala.js" | "Scala Native" => - // The fix assumes a hash collision safe java.util.LinkedHashMap - // implementation. When/if other platforms will have such characteristics - // ujson doesn't need to change. - case _ => - withTimeout { ujson.read(jsonString) } - } - () - } - } } } @@ -387,10 +349,6 @@ object AdvancedTests extends TestSuite { implicit val nodeRW: ReadWriter[Node371] = macroRW[Node371] } - case class Foo416() - object Foo416 { - implicit val rw: ReadWriter[Foo416] = macroRW[Foo416] - } class Thing[T: upickle.default.Writer, V: upickle.default.Writer](t: Option[(V, T)]) { implicitly[upickle.default.Writer[Option[(V, T)]]] } From 7bfd10c5a71e76a6777981892fc5a6bc137dfa29 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 14:29:28 +0800 Subject: [PATCH 03/17] fast string writers --- core/src/upickle/core/RenderUtils.scala | 252 +++++++++++++++--------- core/templates/ElemUtils.scala | 106 ++++++++++ ujson/templates/ElemParser.scala | 2 +- 3 files changed, 268 insertions(+), 92 deletions(-) create mode 100644 core/templates/ElemUtils.scala diff --git a/core/src/upickle/core/RenderUtils.scala b/core/src/upickle/core/RenderUtils.scala index eb486db76..46f68f191 100644 --- a/core/src/upickle/core/RenderUtils.scala +++ b/core/src/upickle/core/RenderUtils.scala @@ -5,6 +5,7 @@ import java.nio.charset.StandardCharsets import scala.annotation.switch object RenderUtils{ + final val hexChars: Array[Int] = { val arr = new Array[Int](128) var i = 0 @@ -15,7 +16,7 @@ object RenderUtils{ } def hex(i: Int): Int = hexChars(i) - private def toHex(nibble: Int): Char = (nibble + (if (nibble >= 10) 87 else 48)).toChar + def toHex(nibble: Int): Char = (nibble + (if (nibble >= 10) 87 else 48)).toChar /** * Attempts to write the given [[CharSequence]] into the given [[ByteBuilder]]. @@ -28,128 +29,164 @@ object RenderUtils{ final def escapeByte(unicodeCharBuilder: upickle.core.CharBuilder, sb: upickle.core.ByteBuilder, s: CharSequence, - unicode: Boolean, + escapeUnicode: Boolean, wrapQuotes: Boolean): Unit = { - var i = 0 val len = s.length - val naiveOutLen = len + (if (wrapQuotes) 2 else 0) // +2 for the start and end quotes - sb.ensureLength(naiveOutLen) - if (wrapQuotes) sb.appendUnsafe('"') - while (i < len) { - (s.charAt(i): @switch) match { - case '"' => escapeSingleByte(sb, i, naiveOutLen, '"') - case '\\' => escapeSingleByte(sb, i, naiveOutLen, '\\') - case '\b' => escapeSingleByte(sb, i, naiveOutLen, 'b') - case '\f' => escapeSingleByte(sb, i, naiveOutLen, 'f') - case '\n' => escapeSingleByte(sb, i, naiveOutLen, 'n') - case '\r' => escapeSingleByte(sb, i, naiveOutLen, 'r') - case '\t' => escapeSingleByte(sb, i, naiveOutLen, 't') - case c => - val notControlChar = c >= ' ' - val notUnicodeChar = c <= '~' - if (notControlChar && notUnicodeChar) sb.append(c) - else if (!notControlChar || (!notUnicodeChar && unicode)) { - escapeSingleByteUnicodeEscape(sb, i, naiveOutLen, c) - } else { + if (wrapQuotes) sb.append('"') + var i = 0 + // Split the loop based on `escapeUnicode`, + // to avoid repeating the check within the hot loop + if (escapeUnicode){ + while (i < len) { + val c = s.charAt(i) + if (!ByteUtils.appendEscapedByte(sb, c, i)) { + if (c < ' ' || c > 127) { ByteUtils.escapeSingleByteUnicodeEscape(sb, i, c) } + else { i = ByteUtils.appendSimpleStringSectionNoUnicode(sb, i, len, s) } + } + i += 1 + } + }else{ + while (i < len) { + val c = s.charAt(i) + if (!ByteUtils.appendEscapedByte(sb, c, i)) { + if (c < ' ') { ByteUtils.escapeSingleByteUnicodeEscape(sb, i, c) } + else if (c <= '~') { i = ByteUtils.appendSimpleStringSectionNoUnicode(sb, i, len, s) } + else { + // If we hit a unicode character, fall back to `escapeSingleByteUnicodeRaw` + // to handle the remainder of the string. Somehow this benchmarks almost + // twice as fast as trying to handle the unicode characters directly in this + // method, for unclear reasons (???) escapeSingleByteUnicodeRaw( - unicodeCharBuilder, sb, s, unicode, i, len, naiveOutLen, wrapQuotes + unicodeCharBuilder, sb, s, false, i, len, wrapQuotes ) return } + } + i += 1 } - i += 1 } - if (wrapQuotes) sb.appendUnsafe('"') + + if (wrapQuotes) sb.append('"') } + @deprecated("Not used, kept for binary compatibility") def escapeSingleByteUnicodeRaw(unicodeCharBuilder: CharBuilder, sb: ByteBuilder, s: CharSequence, - unicode: Boolean, - i: Int, - len: Int, + escapeUnicode: Boolean, + i0: Int, + len0: Int, naiveOutLen: Int, - wrapQuotes: Boolean) = { - unicodeCharBuilder.reset() - escapeChar0(i, naiveOutLen, len, unicodeCharBuilder, s, unicode, wrapQuotes) + wrapQuotes: Boolean): Unit = escapeSingleByteUnicodeRaw( + unicodeCharBuilder, sb, s, escapeUnicode, i0, len0, wrapQuotes + ) - val bytes = unicodeCharBuilder.makeString().getBytes(StandardCharsets.UTF_8) - sb.appendAll(bytes, bytes.length) - } + def escapeSingleByteUnicodeRaw(unicodeCharBuilder: CharBuilder, + sb: ByteBuilder, + s: CharSequence, + escapeUnicode: Boolean, + i0: Int, + len0: Int, + wrapQuotes: Boolean): Unit = { + // Take the unicodeCharBuilder's Array[Char] containing potentially unicode characters and write + // them as UTF-8 bytes in sb's Array[Byte] + // + // Somehow aggregating the unicode string in `unicodeCharBuilder` and then + // looping over the chars here is more than twice as fast as doing the same + // logic in the main `escapeByte` work loop. It's also a lot faster than + // calling `.makeString().getBytes(UTF_8)` on `unicodeCharBuilder`, since + // we don't allocate a temporary String and Array[Byte] + // + // Taken from https://stackoverflow.com/a/9670279 - def escapeSingleByteUnicodeEscape(sb: ByteBuilder, i: Int, naiveOutLen: Int, c: Char) = { - sb.ensureLength(naiveOutLen - i + 5); - sb.appendUnsafeC('\\') - sb.appendUnsafeC('u') - sb.appendUnsafeC(toHex((c >> 12) & 15)) - sb.appendUnsafeC(toHex((c >> 8) & 15)) - sb.appendUnsafeC(toHex((c >> 4) & 15)) - sb.appendUnsafeC(toHex(c & 15)) - } + unicodeCharBuilder.reset() + escapeChar0(i0, len0, unicodeCharBuilder, s, escapeUnicode, wrapQuotes) - def escapeSingleByte(sb: ByteBuilder, i: Int, naiveOutLen: Int, c: Char) = { - sb.ensureLength(naiveOutLen - i + 1); - sb.appendUnsafeC('\\'); - sb.appendUnsafeC(c) + val xs: Array[Char] = unicodeCharBuilder.arr + val len = unicodeCharBuilder.length + sb.ensureLength(len * 3) // worst case + val ys: Array[Byte] = sb.arr + var i = 0 + var j = sb.length // i for chars; j for bytes + while (i < len) { // fill ys with bytes + val c = xs(i) + if (c < 0x80) { + ys(j) = c.toByte + i = i + 1 + j = j + 1 + } else if (c < 0x800) { + ys(j) = (0xc0 | (c >> 6)).toByte + ys(j + 1) = (0x80 | (c & 0x3f)).toByte + i = i + 1 + j = j + 2 + } else if (Character.isHighSurrogate(c)) { + if (len - i < 2) throw new Exception("overflow") + val d = xs(i + 1) + val uc: Int = + if (Character.isLowSurrogate(d)) Character.toCodePoint(c, d) + else throw new Exception("malformed") + ys(j) = (0xf0 | ((uc >> 18))).toByte + ys(j + 1) = (0x80 | ((uc >> 12) & 0x3f)).toByte + ys(j + 2) = (0x80 | ((uc >> 6) & 0x3f)).toByte + ys(j + 3) = (0x80 | (uc & 0x3f)).toByte + i = i + 2 // 2 chars + j = j + 4 + } else if (Character.isLowSurrogate(c)) { + throw new Exception("malformed") + } else { + ys(j) = (0xe0 | (c >> 12)).toByte + ys(j + 1) = (0x80 | ((c >> 6) & 0x3f)).toByte + ys(j + 2) = (0x80 | (c & 0x3f)).toByte + i = i + 1 + j = j + 3 + } + } + sb.length = j } def escapeChar(unicodeCharBuilder: upickle.core.CharBuilder, sb: upickle.core.CharBuilder, s: CharSequence, - unicode: Boolean, + escapeUnicode: Boolean, wrapQuotes: Boolean) = { val len = s.length - val naiveOutLen = len + (if (wrapQuotes) 2 else 0) // +2 for the start and end quotes - sb.ensureLength(naiveOutLen) - if (wrapQuotes) sb.appendUnsafe('"') - escapeChar0(0, naiveOutLen, len, sb, s, unicode, wrapQuotes) + if (wrapQuotes) sb.append('"') + escapeChar0(0, len, sb, s, escapeUnicode, wrapQuotes) } + final def escapeChar0(i0: Int, - naiveOutLen: Int, len: Int, sb: upickle.core.CharBuilder, s: CharSequence, - unicode: Boolean, - wrapQuotes: Boolean) = { + escapeUnicode: Boolean, + wrapQuotes: Boolean): upickle.core.CharBuilder = { var i = i0 - sb.ensureLength(naiveOutLen) - while (i < len) { - (s.charAt(i): @switch) match { - case '"' => escapeSingleChar(sb, naiveOutLen, i, '"') - case '\\' => escapeSingleChar(sb, naiveOutLen, i, '\\') - case '\b' => escapeSingleChar(sb, naiveOutLen, i, 'b') - case '\f' => escapeSingleChar(sb, naiveOutLen, i, 'f') - case '\n' => escapeSingleChar(sb, naiveOutLen, i, 'n') - case '\r' => escapeSingleChar(sb, naiveOutLen, i, 'r') - case '\t' => escapeSingleChar(sb, naiveOutLen, i, 't') - case c => - if (c < ' ' || (c > '~' && unicode)) { - escapeSingleCharUnicodeEscape(naiveOutLen, sb, i, c) - } - else sb.append(c) + + // Split the loop based on `escapeUnicode`, + // to avoid repeating the check within the hot loop + if (escapeUnicode){ + while (i < len) { + val c = s.charAt(i) + if (!CharUtils.appendEscapedChar(sb, c, i)) { + if (c < ' ' || c > 127) { CharUtils.escapeSingleCharUnicodeEscape(sb, i, c) } + else { i = CharUtils.appendSimpleStringSectionNoUnicode(sb, i, len, s) } + } + i += 1 + } + }else{ + while (i < len) { + val c = s.charAt(i) + if (!CharUtils.appendEscapedChar(sb, c, i)) { + if (c < ' ') { CharUtils.escapeSingleCharUnicodeEscape(sb, i, c) } + else { i = CharUtils.appendSimpleStringSection(sb, i, len, s) } + } + i += 1 } - i += 1 } - if (wrapQuotes) sb.appendUnsafe('"') - sb - } - def escapeSingleCharUnicodeEscape(naiveOutLen: Int, sb: CharBuilder, i: Int, c: Char) = { - sb.ensureLength(naiveOutLen - i + 5); - sb.appendUnsafeC('\\') - sb.appendUnsafeC('u') - sb.appendUnsafeC(toHex((c >> 12) & 15)) - sb.appendUnsafeC(toHex((c >> 8) & 15)) - sb.appendUnsafeC(toHex((c >> 4) & 15)) - sb.appendUnsafeC(toHex(c & 15)) - } - - def escapeSingleChar(sb: upickle.core.CharBuilder, - naiveOutLen: Int, - i: Int, - c: Char) = { - sb.ensureLength(naiveOutLen - i + 1); sb.appendUnsafeC('\\'); sb.appendUnsafeC(c) + if (wrapQuotes) sb.append('"') + sb } def intStringSize(x0: Int): Int = { @@ -162,7 +199,7 @@ object RenderUtils{ } var p = -10 var i = 1 - while (i <= 10) { + while(i <= 10){ if (x > p) return i + d p = 10 * p i += 1 @@ -180,11 +217,44 @@ object RenderUtils{ } var p: Long = -10 var i = 1 - while (i <= 18) { + while (i <= 18){ if (x > p) return i + d p = 10 * p i += 1 } 19 + d } + + @deprecated("Not used, kept for binary compatibility") + def escapeSingleChar(sb: upickle.core.CharBuilder, + naiveOutLen: Int, + i: Int, + c: Char) = CharUtils.escapeSingleChar(sb, i, c) + + @deprecated("Not used, kept for binary compatibility") + def escapeSingleByte(sb: upickle.core.ByteBuilder, + naiveOutLen: Int, + i: Int, + c: Char) = ByteUtils.escapeSingleByte(sb, i, c) + + @deprecated("Not used, kept for binary compatibility") + def escapeSingleCharUnicodeEscape(naiveOutLen: Int, sb: CharBuilder, i: Int, c: Char) = + CharUtils.escapeSingleCharUnicodeEscape(sb, i, c) + + @deprecated("Not used, kept for binary compatibility") + def escapeSingleByteUnicodeEscape(sb: ByteBuilder, i: Int, naiveOutLen: Int, c: Char) = { + ByteUtils.escapeSingleByteUnicodeEscape(sb, i, c) + } + + @deprecated("Not used, kept for binary compatibility") + final def escapeChar0(i0: Int, + naiveOutLen: Int, + len: Int, + sb: upickle.core.CharBuilder, + s: CharSequence, + escapeUnicode: Boolean, + wrapQuotes: Boolean): upickle.core.CharBuilder = { + escapeChar0(i0, len, sb, s, escapeUnicode, wrapQuotes) + } + } diff --git a/core/templates/ElemUtils.scala b/core/templates/ElemUtils.scala new file mode 100644 index 000000000..ca6e671da --- /dev/null +++ b/core/templates/ElemUtils.scala @@ -0,0 +1,106 @@ +package upickle.core +import scala.annotation.switch +object ElemUtils{ + + def appendEscapedElem(elemBuilder: ElemBuilder, c: Char, i: Int): Boolean = { + (c: @switch) match { + case '"' => escapeSingleElem(elemBuilder, i, '"'); true + case '\\' => escapeSingleElem(elemBuilder, i, '\\'); true + case '\b' => escapeSingleElem(elemBuilder, i, 'b'); true + case '\f' => escapeSingleElem(elemBuilder, i, 'f'); true + case '\n' => escapeSingleElem(elemBuilder, i, 'n'); true + case '\r' => escapeSingleElem(elemBuilder, i, 'r'); true + case '\t' => escapeSingleElem(elemBuilder, i, 't'); true + case _ => false + } + } + + + def escapeSingleElem(elemBuilder: ElemBuilder, i: Int, c: Char) = { + elemBuilder.ensureLength(2); + val length = elemBuilder.length + val arr = elemBuilder.arr + arr(length) = '\\'.toElem + arr(length + 1) = c.toElem + elemBuilder.length += 2 + } + + + def escapeSingleElemUnicodeEscape(elemBuilder: ElemBuilder, i: Int, c: Char) = { + elemBuilder.ensureLength(6) + val arr = elemBuilder.arr + val length = elemBuilder.length + arr(length) = '\\'.toElem + arr(length + 1) = 'u'.toElem + arr(length + 2) = RenderUtils.toHex((c >> 12) & 15).toElem + arr(length + 3) = RenderUtils.toHex((c >> 8) & 15).toElem + arr(length + 4) = RenderUtils.toHex((c >> 4) & 15).toElem + arr(length + 5) = RenderUtils.toHex(c & 15).toElem + elemBuilder.length += 6 + } + + def appendSimpleStringSection(elemBuilder: ElemBuilder, + i0: Int, + len: Int, + s: CharSequence) = { + elemBuilder.ensureLength(len - i0) + val i = appendSimpleStringSection0(elemBuilder.arr, elemBuilder.length, i0, len, s) + elemBuilder.length = elemBuilder.length + (i - i0) + 1 + i + } + + def appendSimpleStringSectionNoUnicode(elemBuilder: ElemBuilder, + i0: Int, + len: Int, + s: CharSequence) = { + elemBuilder.ensureLength(len - i0) + val i = appendSimpleStringSectionNoUnicode0(elemBuilder.arr, elemBuilder.length, i0, len, s) + elemBuilder.length = elemBuilder.length + (i - i0) + 1 + i + } + + private def appendSimpleStringSection0(arr: Array[Elem], + arrOffset: Int, + i0: Int, + len: Int, + s: CharSequence) = { + var i = i0 + while ( + if (i >= len) false + else { + val c2 = s.charAt(i) + if (c2 < ' ' | c2 == '"' | c2 == '\\') false + else { + arr(arrOffset + i - i0) = c2.toElem + i += 1 + true + } + } + ) () + + i - 1 + } + + private def appendSimpleStringSectionNoUnicode0(arr: Array[Elem], + arrOffset: Int, + i0: Int, + len: Int, + s: CharSequence) = { + var i = i0 + while ( + if (i >= len) false + else { + val c2 = s.charAt(i) + if (c2 < ' ' || c2 > 127 || c2 == '"' || c2 == '\\') false + else { + arr(arrOffset + i - i0) = c2.toElem + i += 1 + true + } + } + ) () + + i - 1 + } + +} \ No newline at end of file diff --git a/ujson/templates/ElemParser.scala b/ujson/templates/ElemParser.scala index 30afcd85a..eff7ea2a9 100644 --- a/ujson/templates/ElemParser.scala +++ b/ujson/templates/ElemParser.scala @@ -79,7 +79,7 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ new upickle.core.CharBuilder(), out, CharBuffer.wrap(Array(elemOps.toInt(getElemSafe(i)).toChar)), - unicode = false, + escapeUnicode = false, true ) val s = "%s got %s" format (msg, out.makeString()) From 5c33531a93fc3f8451fac27004c28eea66e48c25 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 14:39:20 +0800 Subject: [PATCH 04/17] MsgPackWriter improvements --- ujson/src/ujson/JsVisitor.scala | 7 +- upack/src/upack/MsgPackWriter.scala | 233 ++++++++++++++++++---------- 2 files changed, 156 insertions(+), 84 deletions(-) diff --git a/ujson/src/ujson/JsVisitor.scala b/ujson/src/ujson/JsVisitor.scala index f49bed4f7..a56d8fbcb 100644 --- a/ujson/src/ujson/JsVisitor.scala +++ b/ujson/src/ujson/JsVisitor.scala @@ -29,14 +29,15 @@ trait JsVisitor[-T, +J] extends Visitor[T, J]{ def visitInt32(i: Int, index: Int): J = visitFloat64(i, index) def visitInt64(i: Long, index: Int): J = { - if (math.abs(i) > math.pow(2, 53) || i == -9223372036854775808L) visitString(i.toString, index) + if (math.abs(i) > 9007199254740992L /*math.pow(2, 53)*/ || + i == -9223372036854775808L /*Long.MinValue*/) visitString(i.toString, index) else visitFloat64(i.toDouble, index) } def visitUInt64(i: Long, index: Int): J = { - if (i > math.pow(2, 53) || i < 0) visitString(java.lang.Long.toUnsignedString(i), index) + if (i > 9007199254740992L /*math.pow(2, 53)*/ || + i < 0) visitString(java.lang.Long.toUnsignedString(i), index) else visitFloat64(i.toDouble, index) } - def visitFloat64String(s: String, index: Int): J = { visitFloat64StringParts( s, diff --git a/upack/src/upack/MsgPackWriter.scala b/upack/src/upack/MsgPackWriter.scala index 17d9092b4..3af774f72 100644 --- a/upack/src/upack/MsgPackWriter.scala +++ b/upack/src/upack/MsgPackWriter.scala @@ -17,12 +17,16 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea byteBuilder.append(MPK.FixArrMask | length) }else if (length <= 65535){ byteBuilder.ensureLength(3) - byteBuilder.appendUnsafe(MPK.Array16.toByte) - writeUInt16(length) + val arr = byteBuilder.arr; val arrLength = byteBuilder.length + arr(arrLength) = MPK.Array16.toByte + writeUInt16(arr, arrLength, length) + byteBuilder.length += 3 }else { byteBuilder.ensureLength(5) - byteBuilder.appendUnsafe(MPK.Array32.toByte) - writeUInt32(length) + val arr = byteBuilder.arr; val arrLength = byteBuilder.length + arr(arrLength) = MPK.Array32.toByte + writeUInt32(arr, arrLength, length) + byteBuilder.length += 5 } def subVisitor = MsgPackWriter.this def visitValue(v: T, index: Int): Unit = () // do nothing @@ -40,12 +44,16 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea byteBuilder.append(MPK.FixMapMask | length) }else if (length <= 65535){ byteBuilder.ensureLength(3) - byteBuilder.appendUnsafe(MPK.Map16.toByte) - writeUInt16(length) + val arr = byteBuilder.arr; val arrLength = byteBuilder.length + arr(arrLength) = MPK.Map16.toByte + writeUInt16(arr, arrLength, length) + byteBuilder.length += 3 }else { byteBuilder.ensureLength(5) - byteBuilder.appendUnsafe(MPK.Map32.toByte) - writeUInt32(length) + val arr = byteBuilder.arr; val arrLength = byteBuilder.length + arr(arrLength) = MPK.Map32.toByte + writeUInt32(arr, arrLength, length) + byteBuilder.length += 5 } def subVisitor = MsgPackWriter.this def visitKey(index: Int)= MsgPackWriter.this @@ -83,15 +91,21 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea override def visitFloat64(d: Double, index: Int) = { byteBuilder.ensureLength(9) - byteBuilder.append(MPK.Float64) - writeUInt64(java.lang.Double.doubleToLongBits(d)) + val arr = byteBuilder.arr + val length = byteBuilder.length + arr(length) = MPK.Float64.toByte + writeUInt64(arr, length, java.lang.Double.doubleToLongBits(d)) + byteBuilder.length = length + 9 flushElemBuilder() out } override def visitFloat32(d: Float, index: Int) = { byteBuilder.ensureLength(5) - byteBuilder.append(MPK.Float32) - writeUInt32(java.lang.Float.floatToIntBits(d)) + val arr = byteBuilder.arr + val length = byteBuilder.length + arr(length) = MPK.Float32.toByte + writeUInt32(arr, length, java.lang.Float.floatToIntBits(d)) + byteBuilder.length = length + 5 flushElemBuilder() out } @@ -100,35 +114,49 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea if (i <= 127) byteBuilder.append(i) else if (i <= 255){ byteBuilder.ensureLength(2) - byteBuilder.appendUnsafe(MPK.UInt8.toByte) - byteBuilder.appendUnsafe(i.toByte) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.UInt8.toByte + arr(length + 1) = i.toByte + byteBuilder.length = length + 2 } else if(i <= Short.MaxValue){ byteBuilder.ensureLength(3) - byteBuilder.appendUnsafe(MPK.Int16.toByte) - writeUInt16(i) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Int16.toByte + writeUInt16(arr, length, i) + byteBuilder.length = length + 3 } else if (i <= 0xffff){ - byteBuilder.ensureLength(5) - byteBuilder.appendUnsafe(MPK.UInt16.toByte) - writeUInt16(i) + byteBuilder.ensureLength(3) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.UInt16.toByte + writeUInt16(arr, length, i) + byteBuilder.length = length + 3 } else{ byteBuilder.ensureLength(5) - byteBuilder.appendUnsafe(MPK.Int32.toByte) - writeUInt32(i) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Int32.toByte + writeUInt32(arr, length, i) + byteBuilder.length = length + 5 } }else{ if (i >= -32) byteBuilder.append(i | 0xe0) else if(i >= -128){ byteBuilder.ensureLength(2) - byteBuilder.appendUnsafe(MPK.Int8.toByte) - byteBuilder.appendUnsafe(i.toByte) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Int8.toByte + arr(length + 1) = i.toByte + byteBuilder.length = length + 2 }else if (i >= Short.MinValue) { byteBuilder.ensureLength(3) - byteBuilder.appendUnsafe(MPK.Int16.toByte) - writeUInt16(i) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Int16.toByte + writeUInt16(arr, length, i) + byteBuilder.length = length + 3 } else{ byteBuilder.ensureLength(5) - byteBuilder.appendUnsafe(MPK.Int32.toByte) - writeUInt32(i) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Int32.toByte + writeUInt32(arr, length, i) + byteBuilder.length = length + 5 } } flushElemBuilder() @@ -140,12 +168,16 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea visitInt32(i.toInt, index) }else if (i >= 0 && i <= 0xffffffffL){ byteBuilder.ensureLength(5) - byteBuilder.appendUnsafe(MPK.UInt32.toByte) - writeUInt32(i.toInt) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.UInt32.toByte + writeUInt32(arr, length, i.toInt) + byteBuilder.length = length + 5 }else{ byteBuilder.ensureLength(9) - byteBuilder.appendUnsafe(MPK.Int64.toByte) - writeUInt64(i) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Int64.toByte + writeUInt64(arr, length, i) + byteBuilder.length = length + 9 } flushElemBuilder() out @@ -155,8 +187,10 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea if (i >= 0) visitInt64(i, index) else{ byteBuilder.ensureLength(9) - byteBuilder.appendUnsafe(MPK.UInt64.toByte) - writeUInt64(i) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.UInt64.toByte + writeUInt64(arr, length, i) + byteBuilder.length = length + 9 } flushElemBuilder() out @@ -164,67 +198,83 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea override def visitString(s: CharSequence, index: Int) = { val strBytes = s.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8) - val length = strBytes.length - if (length <= 31){ - byteBuilder.ensureLength(1 + length) - byteBuilder.appendUnsafe((MPK.FixStrMask | length).toByte) - } else if (length <= 255){ - byteBuilder.ensureLength(2 + length) - byteBuilder.appendUnsafe(MPK.Str8.toByte) - writeUInt8(length) - }else if (length <= 65535){ - byteBuilder.ensureLength(3 + length) - byteBuilder.appendUnsafe(MPK.Str16.toByte) - writeUInt16(length) + val strLength = strBytes.length + if (strLength <= 31){ + byteBuilder.ensureLength(1 + strLength) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = (MPK.FixStrMask | strLength).toByte + byteBuilder.length = length + 1 + } else if (strLength <= 255){ + byteBuilder.ensureLength(2 + strLength) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Str8.toByte + writeUInt8(arr, length, strLength) + byteBuilder.length = length + 2 + }else if (strLength <= 65535){ + byteBuilder.ensureLength(3 + strLength) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Str16.toByte + writeUInt16(arr, length, strLength) + byteBuilder.length = length + 3 }else { - byteBuilder.ensureLength(5 + length) - byteBuilder.appendUnsafe(MPK.Str32.toByte) - writeUInt32(length) + byteBuilder.ensureLength(5 + strLength) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Str32.toByte + writeUInt32(arr, length, strLength) + byteBuilder.length = length + 5 } - byteBuilder.appendAll(strBytes, length) + byteBuilder.appendAll(strBytes, strLength) flushElemBuilder() out } override def visitBinary(bytes: Array[Byte], offset: Int, len: Int, index: Int) = { if (len <= 255) { byteBuilder.ensureLength(2) - byteBuilder.appendUnsafe(MPK.Bin8.toByte) - writeUInt8(len) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Bin8.toByte + writeUInt8(arr, length, len) + byteBuilder.length = length + 2 } else if (len <= 65535) { byteBuilder.ensureLength(3) - byteBuilder.appendUnsafe(MPK.Bin16.toByte) - writeUInt16(len) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Bin16.toByte + writeUInt16(arr, length, len) + byteBuilder.length = length + 3 } else { byteBuilder.ensureLength(5) - byteBuilder.appendUnsafe(MPK.Bin32.toByte) - writeUInt32(len) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Bin32.toByte + writeUInt32(arr, length, len) + byteBuilder.length = length + 5 } byteBuilder.appendAll(bytes, offset, len) flushElemBuilder() out } - def writeUInt8(i: Int) = byteBuilder.appendUnsafe(i.toByte) - def writeUInt16(i: Int) = { - byteBuilder.appendUnsafe(((i >> 8) & 0xff).toByte) - byteBuilder.appendUnsafe(((i >> 0) & 0xff).toByte) + + def writeUInt8(arr: Array[Byte], length: Int, i: Int): Unit = arr(length + 1) = i.toByte + + def writeUInt16(arr: Array[Byte], length: Int, i: Int): Unit = { + arr(length + 1) = ((i >> 8) & 0xff).toByte + arr(length + 2) = ((i >> 0) & 0xff).toByte } - def writeUInt32(i: Int) = { - byteBuilder.appendUnsafe(((i >> 24) & 0xff).toByte) - byteBuilder.appendUnsafe(((i >> 16) & 0xff).toByte) - byteBuilder.appendUnsafe(((i >> 8) & 0xff).toByte) - byteBuilder.appendUnsafe(((i >> 0) & 0xff).toByte) + def writeUInt32(arr: Array[Byte], length: Int, i: Int): Unit = { + arr(length + 1) = ((i >> 24) & 0xff).toByte + arr(length + 2) = ((i >> 16) & 0xff).toByte + arr(length + 3) = ((i >> 8) & 0xff).toByte + arr(length + 4) = ((i >> 0) & 0xff).toByte } - def writeUInt64(i: Long) = { - byteBuilder.appendUnsafe(((i >> 56) & 0xff).toByte ) - byteBuilder.appendUnsafe(((i >> 48) & 0xff).toByte ) - byteBuilder.appendUnsafe(((i >> 40) & 0xff).toByte ) - byteBuilder.appendUnsafe(((i >> 32) & 0xff).toByte ) - byteBuilder.appendUnsafe(((i >> 24) & 0xff).toByte ) - byteBuilder.appendUnsafe(((i >> 16) & 0xff).toByte ) - byteBuilder.appendUnsafe(((i >> 8) & 0xff).toByte ) - byteBuilder.appendUnsafe(((i >> 0) & 0xff).toByte ) + def writeUInt64(arr: Array[Byte], length: Int, i: Long): Unit = { + arr(length + 1) = ((i >> 56) & 0xff).toByte + arr(length + 2) = ((i >> 48) & 0xff).toByte + arr(length + 3) = ((i >> 40) & 0xff).toByte + arr(length + 4) = ((i >> 32) & 0xff).toByte + arr(length + 5) = ((i >> 24) & 0xff).toByte + arr(length + 6) = ((i >> 16) & 0xff).toByte + arr(length + 7) = ((i >> 8) & 0xff).toByte + arr(length + 8) = ((i >> 0) & 0xff).toByte } def visitExt(tag: Byte, bytes: Array[Byte], offset: Int, len: Int, index: Int) = { @@ -237,16 +287,22 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea case _ => if (len <= 255){ byteBuilder.ensureLength(2) - byteBuilder.appendUnsafe(MPK.Ext8.toByte) - writeUInt8(len) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Ext8.toByte + writeUInt8(arr, length, len) + byteBuilder.length = length + 2 }else if (len <= 65535){ byteBuilder.ensureLength(3) - byteBuilder.appendUnsafe(MPK.Ext16.toByte) - writeUInt16(len) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.Ext16.toByte + writeUInt16(arr, length, len) + byteBuilder.length = length + 3 }else{ byteBuilder.ensureLength(5) - writeUInt32(len) - byteBuilder.appendUnsafe(MPK.Ext32.toByte) + val arr = byteBuilder.arr; val length = byteBuilder.length + writeUInt32(arr, length, len) + arr(length) = MPK.Ext32.toByte + byteBuilder.length = length + 5 } } byteBuilder.append(tag) @@ -257,9 +313,24 @@ class MsgPackWriter[T <: java.io.OutputStream](out: T = new ByteArrayOutputStrea def visitChar(s: Char, index: Int) = { byteBuilder.ensureLength(3) - byteBuilder.appendUnsafe(MPK.UInt16.toByte) - writeUInt16(s) + val arr = byteBuilder.arr; val length = byteBuilder.length + arr(length) = MPK.UInt16.toByte + writeUInt16(arr, length, s) + byteBuilder.length = length + 3 flushElemBuilder() out } + + + @deprecated("Not used, kept for binary compatibility") + def writeUInt8(i: Int): Unit = writeUInt8(byteBuilder.arr, byteBuilder.length, i) + + @deprecated("Not used, kept for binary compatibility") + def writeUInt16(i: Int): Unit = writeUInt16(byteBuilder.arr, byteBuilder.length, i) + + @deprecated("Not used, kept for binary compatibility") + def writeUInt32(i: Int): Unit = writeUInt32(byteBuilder.arr, byteBuilder.length, i) + + @deprecated("Not used, kept for binary compatibility") + def writeUInt64(i: Long): Unit = writeUInt64(byteBuilder.arr, byteBuilder.length, i) } From cb59f0bb09e9edcee7598ee255bcf845de6415fb Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 14:58:53 +0800 Subject: [PATCH 05/17] BufferingElemParser changes --- core/src/upickle/core/Exceptions.scala | 9 ++++ core/templates/BufferingElemParser.scala | 45 ++++++++++++++++--- ujson/src/ujson/Exceptions.scala | 6 +-- ujson/templates/ElemParser.scala | 5 +-- upickle/test/src/upickle/FailureTests.scala | 2 +- .../src/upickle/example/ExampleTests.scala | 4 +- 6 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 core/src/upickle/core/Exceptions.scala diff --git a/core/src/upickle/core/Exceptions.scala b/core/src/upickle/core/Exceptions.scala new file mode 100644 index 000000000..5663a83d5 --- /dev/null +++ b/core/src/upickle/core/Exceptions.scala @@ -0,0 +1,9 @@ +package upickle.core + +sealed trait ParsingFailedException extends Exception + +case class ParseException(clue: String, index: Int) + extends Exception(clue + " at index " + index) with ParsingFailedException + +case class IncompleteParseException(msg: String) + extends Exception(msg) with ParsingFailedException diff --git a/core/templates/BufferingElemParser.scala b/core/templates/BufferingElemParser.scala index 6533abbc4..ce5264065 100644 --- a/core/templates/BufferingElemParser.scala +++ b/core/templates/BufferingElemParser.scala @@ -54,7 +54,7 @@ trait BufferingElemParser{ def getLastIdx = lastIdx def getElemSafe(i: Int): Elem = { - requestUntil(i) + requestUntilOrThrow(i) buffer(i - firstIdx) } def getElemUnsafe(i: Int): Elem = { @@ -62,15 +62,29 @@ trait BufferingElemParser{ } def sliceString(i: Int, k: Int): String = { - requestUntil(k - 1) + requestUntilOrThrow(k - 1) ElemOps.newString(buffer, i - firstIdx, k - i) } def sliceArr(i: Int, n: Int): (Array[Elem], Int, Int) = { - requestUntil(i + n - 1) + requestUntilOrThrow(i + n - 1) (buffer, i - firstIdx, n) } + /** + * A fast-path to check whether an index can be safely accessed, before calling + * [[getElemUnsafe]]. Together, it is similar to calling [[getElemSafe]], except + * this returns the new safeIndex which the caller can then use to call + * [[getElemUnsafe]] multiple times before needing to call this again. + * + */ + def requestUntilOrThrow(j: Int):Unit = checkSafeIndex(j) + def checkSafeIndex(j: Int): Int = { + val newSafeIndex = requestUntilGetSafeIndex(j) + if (newSafeIndex == j) throw new IncompleteParseException("exhausted input") + newSafeIndex + } + /** * Copies the non-dropped Elems in the current [[buffer]] to the start of either * the current [[buffer]], or a newly-allocated larger [[buffer]] if necessary. @@ -119,16 +133,34 @@ trait BufferingElemParser{ else requestUntil0(until) } + /** + * Used to ask for data up to a certain index, as a best effort (unlike + * [[requestUntil]]), returning the "safe index" which it was actually able + * to fetch data for. This is used so the caller can use the safe index to + * know how far it is able to run [[getElemUnsafe]] calls without further + * checks, improving performance over calling [[getElemSafe]] every time + * which performs additional checks and logic + */ + protected def requestUntilGetSafeIndex(until: Int): Int = { + if (until < lastIdx) lastIdx + else if (until >= knownEof) knownEof + else { + val newDone = requestUntil0(until) + if (newDone) knownEof + else lastIdx + } + } + private def requestUntil0(until: Int) = { val untilBufferOffset = until - firstIdx if (buffer != null && untilBufferOffset >= buffer.length) growBuffer(until) + val bufferOffset = lastIdx - firstIdx val (newBuffer, newDone, n) = readDataIntoBuffer(buffer, bufferOffset) buffer = newBuffer - - lastIdx = lastIdx + n - if (newDone) knownEof = until + if (n != -1) lastIdx = lastIdx + n + if (newDone) knownEof = lastIdx newDone } @@ -140,7 +172,6 @@ trait BufferingElemParser{ def unsafeCharSeqForRange(start: Int, length: Int) = { new WrapElemArrayCharSeq(buffer, start - firstIdx, length) } - def appendElemsToBuilder(elems: ElemBuilder, elemsStart: Int, elemsLength: Int) = { elems.appendAll(buffer, elemsStart - firstIdx, elemsLength) } diff --git a/ujson/src/ujson/Exceptions.scala b/ujson/src/ujson/Exceptions.scala index a20c090a0..fefb70e32 100644 --- a/ujson/src/ujson/Exceptions.scala +++ b/ujson/src/ujson/Exceptions.scala @@ -1,10 +1,10 @@ package ujson - +@deprecated sealed trait ParsingFailedException extends Exception - +@deprecated case class ParseException(clue: String, index: Int) extends Exception(clue + " at index " + index) with ParsingFailedException - +@deprecated case class IncompleteParseException(msg: String) extends Exception(msg) with ParsingFailedException diff --git a/ujson/templates/ElemParser.scala b/ujson/templates/ElemParser.scala index eff7ea2a9..115a72f3f 100644 --- a/ujson/templates/ElemParser.scala +++ b/ujson/templates/ElemParser.scala @@ -20,9 +20,6 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ private[this] val elemOps = upickle.core.ElemOps private[this] val outputBuilder = new upickle.core.ElemBuilder() - def requestUntilOrThrow(i: Int) = { - if (requestUntil(i)) throw new IncompleteParseException("exhausted input") - } override def getElemSafe(i: Int): Elem = { requestUntilOrThrow(i) getElemUnsafe(i) @@ -83,7 +80,7 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ true ) val s = "%s got %s" format (msg, out.makeString()) - throw ParseException(s, i) + throw upickle.core.ParseException(s, i) } diff --git a/upickle/test/src/upickle/FailureTests.scala b/upickle/test/src/upickle/FailureTests.scala index 4ccd5bc84..7f0618d0e 100644 --- a/upickle/test/src/upickle/FailureTests.scala +++ b/upickle/test/src/upickle/FailureTests.scala @@ -2,7 +2,7 @@ package upickle import utest._ import upickle.legacy.read -import ujson.{IncompleteParseException, ParseException} +import upickle.core.{IncompleteParseException, ParseException} import upickle.core.AbortException case class Fee(i: Int, s: String) object Fee{ diff --git a/upickle/test/src/upickle/example/ExampleTests.scala b/upickle/test/src/upickle/example/ExampleTests.scala index 1079fe536..3957764b7 100644 --- a/upickle/test/src/upickle/example/ExampleTests.scala +++ b/upickle/test/src/upickle/example/ExampleTests.scala @@ -5,8 +5,8 @@ import java.io.StringWriter import upickle.{TestUtil, default} import utest._ import upickle.default.{macroRW, ReadWriter => RW} -import ujson.{IncompleteParseException, ParseException, Readable} -import ujson.{BytesRenderer, Value, StringRenderer} +import upickle.core.{IncompleteParseException, ParseException} +import ujson.{BytesRenderer, Value, StringRenderer, Readable} import upickle.core.{NoOpVisitor, Visitor} object Simple { From ec6d100169068d0f8d8f7fdd8052895d3e5b4b5c Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 18 Mar 2023 20:24:39 +0800 Subject: [PATCH 06/17] properly make longReader accept non-trivial strings --- implicits/src/upickle/implicits/Readers.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/implicits/src/upickle/implicits/Readers.scala b/implicits/src/upickle/implicits/Readers.scala index 494f4cc51..3b6fd6432 100644 --- a/implicits/src/upickle/implicits/Readers.scala +++ b/implicits/src/upickle/implicits/Readers.scala @@ -171,7 +171,7 @@ trait Readers extends upickle.core.Types implicit val LongReader: Reader[Long] = new NumericReader[Long] { override def expectedMsg = "expected number" - override def visitString(d: CharSequence, index: Int) = upickle.core.ParseUtils.parseLong(d, 0, d.length()) + override def visitString(d: CharSequence, index: Int) = visitFloat64String(d.toString, index) override def visitInt32(d: Int, index: Int) = d.toLong override def visitInt64(d: Long, index: Int) = d.toLong override def visitUInt64(d: Long, index: Int) = d.toLong From 0b37e9579a38241f963925ba0ac21c353fbe874e Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 09:15:41 +0800 Subject: [PATCH 07/17] de-virtualize CaseClassReader.write0 --- .../upickle/implicits/internal/Macros.scala | 8 ++++++++ implicits/src-3/upickle/implicits/Writers.scala | 16 +++++++++++++++- .../upickle/implicits/CaseClassReadWriters.scala | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/implicits/src-2/upickle/implicits/internal/Macros.scala b/implicits/src-2/upickle/implicits/internal/Macros.scala index 2a28288c8..15a341ddb 100644 --- a/implicits/src-2/upickle/implicits/internal/Macros.scala +++ b/implicits/src-2/upickle/implicits/internal/Macros.scala @@ -369,6 +369,14 @@ object Macros { .foldLeft[Tree](q"0"){case (prev, next) => q"$prev + $next"} } } + override def write0[R](out: _root_.upickle.core.Visitor[_, R], v: $targetType): R = { + if (v == null) out.visitNull(-1) + else { + val ctx = out.visitObject(length(v), true, -1) + ..${(0 until rawArgs.length).map(write)} + ctx.visitEnd(-1) + } + } def writeToObject[R](ctx: _root_.upickle.core.ObjVisitor[_, R], v: $targetType): Unit = { ..${(0 until rawArgs.length).map(write)} diff --git a/implicits/src-3/upickle/implicits/Writers.scala b/implicits/src-3/upickle/implicits/Writers.scala index 570dee047..d14f4a7af 100644 --- a/implicits/src-3/upickle/implicits/Writers.scala +++ b/implicits/src-3/upickle/implicits/Writers.scala @@ -4,7 +4,7 @@ import upickle.core.Annotator import deriving.Mirror import scala.reflect.ClassTag -import upickle.core.{ Visitor, ObjVisitor, Annotator } +import upickle.core.{Annotator, ObjVisitor, Visitor} trait WritersVersionSpecific extends MacrosCommon @@ -19,6 +19,20 @@ trait WritersVersionSpecific def writer = new CaseClassWriter[T] { def length(v: T) = macros.writeLength[T](outerThis, v) + override def write0[R](out: Visitor[_, R], v: V): R = { + if (v == null) out.visitNull(-1) + else { + val ctx = out.visitObject(length(v), true, -1) + macros.writeSnippets[R, T, Tuple.Map[m.MirroredElemTypes, Writer]]( + outerThis, + this, + v, + ctx + ) + ctx.visitEnd(-1) + } + } + def writeToObject[R](ctx: _root_.upickle.core.ObjVisitor[_, R], v: T): Unit = macros.writeSnippets[R, T, Tuple.Map[m.MirroredElemTypes, Writer]]( outerThis, diff --git a/implicits/src/upickle/implicits/CaseClassReadWriters.scala b/implicits/src/upickle/implicits/CaseClassReadWriters.scala index 2f3a27879..6fa0dd814 100644 --- a/implicits/src/upickle/implicits/CaseClassReadWriters.scala +++ b/implicits/src/upickle/implicits/CaseClassReadWriters.scala @@ -21,6 +21,7 @@ trait CaseClassReadWriters extends upickle.core.Types{ trait CaseClassWriter[V] extends ObjectWriter[V] { def length(v: V): Int + @deprecated("Not used, left for binary compatibility") def writeToObject[R](ctx: ObjVisitor[_, R], v: V): Unit def write0[R](out: Visitor[_, R], v: V): R = { From 6329d730113e6804a431c86ba2a92a5b3900b4af Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 15:30:36 +0800 Subject: [PATCH 08/17] avoid allocating a objectAttributeKeyWriteMap lambda when writing case classes --- .../upickle/implicits/internal/Macros.scala | 5 ++--- .../src-3/upickle/implicits/macros.scala | 3 +-- .../implicits/CaseClassReadWriters.scala | 19 ++++++++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/implicits/src-2/upickle/implicits/internal/Macros.scala b/implicits/src-2/upickle/implicits/internal/Macros.scala index 15a341ddb..db3b5b6dd 100644 --- a/implicits/src-2/upickle/implicits/internal/Macros.scala +++ b/implicits/src-2/upickle/implicits/internal/Macros.scala @@ -345,10 +345,9 @@ object Macros { def write(i: Int) = { val snippet = q""" - this.writeSnippet[R, ${argTypes(i)}]( - ${c.prefix}.objectAttributeKeyWriteMap, + this.writeSnippetMappedName[R, ${argTypes(i)}]( ctx, - ${mappedArgs(i)}, + ${c.prefix}.objectAttributeKeyWriteMap(${mappedArgs(i)}), implicitly[${c.prefix}.Writer[${argTypes(i)}]], v.${TermName(rawArgs(i))} ) diff --git a/implicits/src-3/upickle/implicits/macros.scala b/implicits/src-3/upickle/implicits/macros.scala index da4fdde64..adb5224e9 100644 --- a/implicits/src-3/upickle/implicits/macros.scala +++ b/implicits/src-3/upickle/implicits/macros.scala @@ -144,9 +144,8 @@ def writeSnippetsImpl[R, T, WS <: Tuple](thisOuter: Expr[upickle.core.Types with val select = Select.unique(v.asTerm, rawLabel.name).asExprOf[Any] val snippet = '{ ${self}.writeSnippet[R, tpe]( - ${thisOuter}.objectAttributeKeyWriteMap, ${ctx}, - ${Expr(label)}, + ${thisOuter}.objectAttributeKeyWriteMap(${Expr(label)}), summonInline[Tuple.Elem[WS, index]], ${select}, ) diff --git a/implicits/src/upickle/implicits/CaseClassReadWriters.scala b/implicits/src/upickle/implicits/CaseClassReadWriters.scala index 6fa0dd814..3226d3115 100644 --- a/implicits/src/upickle/implicits/CaseClassReadWriters.scala +++ b/implicits/src/upickle/implicits/CaseClassReadWriters.scala @@ -24,6 +24,7 @@ trait CaseClassReadWriters extends upickle.core.Types{ @deprecated("Not used, left for binary compatibility") def writeToObject[R](ctx: ObjVisitor[_, R], v: V): Unit + @scala.annotation.nowarn def write0[R](out: Visitor[_, R], v: V): R = { if (v == null) out.visitNull(-1) else { @@ -33,16 +34,24 @@ trait CaseClassReadWriters extends upickle.core.Types{ } } + def writeSnippetMappedName[R, V](ctx: _root_.upickle.core.ObjVisitor[_, R], + mappedArgsI: CharSequence, + w: Any, + value: Any) = { + val keyVisitor = ctx.visitKey(-1) + ctx.visitKeyValue( + keyVisitor.visitString(mappedArgsI, -1) + ) + ctx.narrow.visitValue(w.asInstanceOf[Writer[Any]].write(ctx.subVisitor, value), -1) + } + + @deprecated("Not used, left for binary compatibility") def writeSnippet[R, V](objectAttributeKeyWriteMap: CharSequence => CharSequence, ctx: _root_.upickle.core.ObjVisitor[_, R], mappedArgsI: String, w: Any, value: Any) = { - val keyVisitor = ctx.visitKey(-1) - ctx.visitKeyValue( - keyVisitor.visitString(objectAttributeKeyWriteMap(mappedArgsI), -1) - ) - ctx.narrow.visitValue(w.asInstanceOf[Writer[Any]].write(ctx.subVisitor, value), -1) + writeSnippetMappedName(ctx, objectAttributeKeyWriteMap(mappedArgsI), w, value) } } From 600979c7969dd69f5f39f87577f4ae968bc70078 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 16:38:24 +0800 Subject: [PATCH 09/17] use checkSafeIndex for string parsing --- implicits/src/upickle/implicits/Readers.scala | 2 +- ujson/templates/ElemParser.scala | 58 +++++++++++-------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/implicits/src/upickle/implicits/Readers.scala b/implicits/src/upickle/implicits/Readers.scala index 3b6fd6432..2606b1d20 100644 --- a/implicits/src/upickle/implicits/Readers.scala +++ b/implicits/src/upickle/implicits/Readers.scala @@ -152,7 +152,7 @@ trait Readers extends upickle.core.Types implicit val CharReader: Reader[Char] = new NumericReader[Char] { override def expectedMsg = "expected char" - override def visitString(d: CharSequence, index: Int) = d.charAt(0) + override def visitString(d: CharSequence, index: Int) = d.toString.charAt(0) override def visitChar(d: Char, index: Int) = d override def visitInt32(d: Int, index: Int) = d.toChar override def visitInt64(d: Long, index: Int) = d.toChar diff --git a/ujson/templates/ElemParser.scala b/ujson/templates/ElemParser.scala index 115a72f3f..3aeda2802 100644 --- a/ujson/templates/ElemParser.scala +++ b/ujson/templates/ElemParser.scala @@ -512,61 +512,71 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ * This method expects the data to be in UTF-16 and accesses it as * chars. */ - protected[this] final def parseStringSimple(i: Int): Int = { + protected[this] final def parseStringSimple(i: Int, safeIndex0: Int = 0): Int = { var j = i - var c = elemOps.toUnsignedInt(getElemSafe(j)) - while (c != '"') { - if (c < ' ') die(j, s"control char (${c}) in string") - if (c == '\\' || c > 127) return -1 - j - j += 1 - c = elemOps.toUnsignedInt(getElemSafe(j)) + var safeIndex = safeIndex0 + while (true) { + // request batches of `Elem` at a time, so within each batch we can use + // `getElemUnsafe` to avoid making a `requestUntil` for every single + // element we fetch + if (j >= safeIndex) safeIndex = checkSafeIndex(j) + val c = getElemUnsafe(j) + (c: @switch) match { + case '"' => return j + 1 + case 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | + 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | + 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | + 30 | 31 => die(j, s"control char (${upickle.core.ElemOps.toUnsignedInt(c)}) in string") + case '\\' => return -1 - j + case _ => j += 1 + } } - j + 1 + ??? } /** * Parse a string that is known to have escape sequences. */ - protected[this] final def parseStringComplex(i0: Int): Int = { + protected[this] final def parseStringComplex(i0: Int, safeIndex0: Int = 0): Int = { + val elemOps = upickle.core.ElemOps var i = i0 - var c = elemOps.toUnsignedInt(getElemSafe(i)) - while (c != '"') { - - if (c < ' ') die(i, s"control char (${c}) in string") + var safeIndex = safeIndex0 + while (true) { + val c = elemOps.toUnsignedInt(getElemSafe(i)) + if (c == '"') return i + 1 + else if (c < ' ') die(i, s"control char (${c}) in string") else if (c == '\\') { - (getElemSafe(i + 1): @switch) match { + if (i + 1 >= safeIndex) safeIndex = checkSafeIndex(i + 1) + (getElemUnsafe(i + 1): @switch) match { case 'b' => { outputBuilder.append('\b'); i += 2 } case 'f' => { outputBuilder.append('\f'); i += 2 } case 'n' => { outputBuilder.append('\n'); i += 2 } case 'r' => { outputBuilder.append('\r'); i += 2 } case 't' => { outputBuilder.append('\t'); i += 2 } - case '"' => { outputBuilder.append('"'); i += 2 } case '/' => { outputBuilder.append('/'); i += 2 } case '\\' => { outputBuilder.append('\\'); i += 2 } // if there's a problem then descape will explode case 'u' => - val d = descape(i) - outputBuilder.appendC(d) - + outputBuilder.appendC(descape(i)) i += 6 case c => die(i + 1, s"illegal escape sequence after \\") } - } else { + }else{ // this case is for "normal" code points that are just one Char. // // we don't have to worry about surrogate pairs, since those // will all be in the ranges D800–DBFF (high surrogates) or // DC00–DFFF (low surrogates). - outputBuilder.append(c) - i += 1 + val k = parseStringSimple(i, safeIndex) + val normalizedK = if (k >= 0) k else -k + appendElemsToBuilder(outputBuilder, i, normalizedK - i - 1) + i = normalizedK - 1 } - c = elemOps.toUnsignedInt(getElemSafe(i)) } - - i + 1 + ??? } /** From e1b3c4e377407bd332aadbb0a1eadf21964a0ef3 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 19:11:23 +0800 Subject: [PATCH 10/17] fix tests --- core/src/upickle/core/Exceptions.scala | 9 --------- core/templates/BufferingElemParser.scala | 20 +++---------------- ujson/src/ujson/Exceptions.scala | 6 +++--- ujson/templates/ElemParser.scala | 17 +++++++++++++++- upickle/test/src/upickle/FailureTests.scala | 2 +- .../src/upickle/example/ExampleTests.scala | 2 +- 6 files changed, 24 insertions(+), 32 deletions(-) delete mode 100644 core/src/upickle/core/Exceptions.scala diff --git a/core/src/upickle/core/Exceptions.scala b/core/src/upickle/core/Exceptions.scala deleted file mode 100644 index 5663a83d5..000000000 --- a/core/src/upickle/core/Exceptions.scala +++ /dev/null @@ -1,9 +0,0 @@ -package upickle.core - -sealed trait ParsingFailedException extends Exception - -case class ParseException(clue: String, index: Int) - extends Exception(clue + " at index " + index) with ParsingFailedException - -case class IncompleteParseException(msg: String) - extends Exception(msg) with ParsingFailedException diff --git a/core/templates/BufferingElemParser.scala b/core/templates/BufferingElemParser.scala index ce5264065..5ae868140 100644 --- a/core/templates/BufferingElemParser.scala +++ b/core/templates/BufferingElemParser.scala @@ -54,7 +54,7 @@ trait BufferingElemParser{ def getLastIdx = lastIdx def getElemSafe(i: Int): Elem = { - requestUntilOrThrow(i) + requestUntil(i) buffer(i - firstIdx) } def getElemUnsafe(i: Int): Elem = { @@ -62,29 +62,15 @@ trait BufferingElemParser{ } def sliceString(i: Int, k: Int): String = { - requestUntilOrThrow(k - 1) + requestUntil(k - 1) ElemOps.newString(buffer, i - firstIdx, k - i) } def sliceArr(i: Int, n: Int): (Array[Elem], Int, Int) = { - requestUntilOrThrow(i + n - 1) + requestUntil(i + n - 1) (buffer, i - firstIdx, n) } - /** - * A fast-path to check whether an index can be safely accessed, before calling - * [[getElemUnsafe]]. Together, it is similar to calling [[getElemSafe]], except - * this returns the new safeIndex which the caller can then use to call - * [[getElemUnsafe]] multiple times before needing to call this again. - * - */ - def requestUntilOrThrow(j: Int):Unit = checkSafeIndex(j) - def checkSafeIndex(j: Int): Int = { - val newSafeIndex = requestUntilGetSafeIndex(j) - if (newSafeIndex == j) throw new IncompleteParseException("exhausted input") - newSafeIndex - } - /** * Copies the non-dropped Elems in the current [[buffer]] to the start of either * the current [[buffer]], or a newly-allocated larger [[buffer]] if necessary. diff --git a/ujson/src/ujson/Exceptions.scala b/ujson/src/ujson/Exceptions.scala index fefb70e32..a20c090a0 100644 --- a/ujson/src/ujson/Exceptions.scala +++ b/ujson/src/ujson/Exceptions.scala @@ -1,10 +1,10 @@ package ujson -@deprecated + sealed trait ParsingFailedException extends Exception -@deprecated + case class ParseException(clue: String, index: Int) extends Exception(clue + " at index " + index) with ParsingFailedException -@deprecated + case class IncompleteParseException(msg: String) extends Exception(msg) with ParsingFailedException diff --git a/ujson/templates/ElemParser.scala b/ujson/templates/ElemParser.scala index 3aeda2802..eb1b6a56c 100644 --- a/ujson/templates/ElemParser.scala +++ b/ujson/templates/ElemParser.scala @@ -20,6 +20,21 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ private[this] val elemOps = upickle.core.ElemOps private[this] val outputBuilder = new upickle.core.ElemBuilder() + /** + * A fast-path to check whether an index can be safely accessed, before calling + * [[getElemUnsafe]]. Together, it is similar to calling [[getElemSafe]], except + * this returns the new safeIndex which the caller can then use to call + * [[getElemUnsafe]] multiple times before needing to call this again. + * + */ + def requestUntilOrThrow(j: Int): Unit = checkSafeIndex(j) + + def checkSafeIndex(j: Int): Int = { + val newSafeIndex = requestUntilGetSafeIndex(j) + if (newSafeIndex == j) throw new IncompleteParseException("exhausted input") + newSafeIndex + } + override def getElemSafe(i: Int): Elem = { requestUntilOrThrow(i) getElemUnsafe(i) @@ -80,7 +95,7 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ true ) val s = "%s got %s" format (msg, out.makeString()) - throw upickle.core.ParseException(s, i) + throw ujson.ParseException(s, i) } diff --git a/upickle/test/src/upickle/FailureTests.scala b/upickle/test/src/upickle/FailureTests.scala index 7f0618d0e..4ccd5bc84 100644 --- a/upickle/test/src/upickle/FailureTests.scala +++ b/upickle/test/src/upickle/FailureTests.scala @@ -2,7 +2,7 @@ package upickle import utest._ import upickle.legacy.read -import upickle.core.{IncompleteParseException, ParseException} +import ujson.{IncompleteParseException, ParseException} import upickle.core.AbortException case class Fee(i: Int, s: String) object Fee{ diff --git a/upickle/test/src/upickle/example/ExampleTests.scala b/upickle/test/src/upickle/example/ExampleTests.scala index 3957764b7..91340d518 100644 --- a/upickle/test/src/upickle/example/ExampleTests.scala +++ b/upickle/test/src/upickle/example/ExampleTests.scala @@ -5,7 +5,7 @@ import java.io.StringWriter import upickle.{TestUtil, default} import utest._ import upickle.default.{macroRW, ReadWriter => RW} -import upickle.core.{IncompleteParseException, ParseException} +import ujson.{IncompleteParseException, ParseException} import ujson.{BytesRenderer, Value, StringRenderer, Readable} import upickle.core.{NoOpVisitor, Visitor} From 84827e0b98f48ca8b859f2b281d219f07976f42a Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 19:53:45 +0800 Subject: [PATCH 11/17] fix tests by introducing WrapElemArrayCharSeq.NonUnicode --- core/templates/BufferingElemParser.scala | 3 +++ core/templates/WrapElemArrayCharSeq.scala | 33 +++++++++++++++++++++-- ujson/templates/ElemParser.scala | 2 +- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/core/templates/BufferingElemParser.scala b/core/templates/BufferingElemParser.scala index 5ae868140..c14fdb85a 100644 --- a/core/templates/BufferingElemParser.scala +++ b/core/templates/BufferingElemParser.scala @@ -158,6 +158,9 @@ trait BufferingElemParser{ def unsafeCharSeqForRange(start: Int, length: Int) = { new WrapElemArrayCharSeq(buffer, start - firstIdx, length) } + def unsafeNonUnicodeCharSeqForRange(start: Int, length: Int) = { + new WrapElemArrayCharSeq.NonUnicode(buffer, start - firstIdx, length) + } def appendElemsToBuilder(elems: ElemBuilder, elemsStart: Int, elemsLength: Int) = { elems.appendAll(buffer, elemsStart - firstIdx, elemsLength) } diff --git a/core/templates/WrapElemArrayCharSeq.scala b/core/templates/WrapElemArrayCharSeq.scala index 9c8a157bf..c162ce1bb 100644 --- a/core/templates/WrapElemArrayCharSeq.scala +++ b/core/templates/WrapElemArrayCharSeq.scala @@ -12,10 +12,10 @@ class WrapElemArrayCharSeq(arr: Array[Elem], start: Int, length0: Int) extends C // start < arr.length && start + length0 <= arr.length && start >= 0 && length0 >= 0, // s"WrapElemArrayCharSeq(${arr.length}, $start, $length0)" // ) - def length() = length0 + def length() = toString.length def charAt(index: Int) = { - arr(index + start).toChar + toString.charAt(index) } def subSequence(newStart: Int, newEnd: Int) = { @@ -32,3 +32,32 @@ class WrapElemArrayCharSeq(arr: Array[Elem], start: Int, length0: Int) extends C } } } + +object WrapElemArrayCharSeq{ + class NonUnicode(arr: Array[Elem], start: Int, length0: Int) extends CharSequence { + // assert( + // start < arr.length && start + length0 <= arr.length && start >= 0 && length0 >= 0, + // s"WrapElemArrayCharSeq(${arr.length}, $start, $length0)" + // ) + def length() = length0 + + def charAt(index: Int) = { + arr(index + start).toChar + } + + def subSequence(newStart: Int, newEnd: Int) = { + new WrapElemArrayCharSeq(arr, start + newStart, newEnd - newStart) + } + + private[this] var toString0: String = null + + override def toString = { + if (toString0 != null) toString0 + else { + val res = ElemOps.newString(arr, start, length0) + toString0 = res + res + } + } + } +} \ No newline at end of file diff --git a/ujson/templates/ElemParser.scala b/ujson/templates/ElemParser.scala index eb1b6a56c..57eb6d737 100644 --- a/ujson/templates/ElemParser.scala +++ b/ujson/templates/ElemParser.scala @@ -168,7 +168,7 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ i: Int, j: Int) = { facade.visitFloat64StringParts( - unsafeCharSeqForRange(i, j - i), + unsafeNonUnicodeCharSeqForRange(i, j - i), decIndex, expIndex, i From fc104f52b9822ccc5a405f4d9dd9465ed938c876 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 20:12:20 +0800 Subject: [PATCH 12/17] make parser use specialized visitFloat64ElemParts callback for handling numbers --- core/src/upickle/core/Visitor.scala | 6 ++ core/templates/BufferingElemParser.scala | 2 + core/templates/ElemUtils.scala | 74 +++++++++++++++++++ implicits/src/upickle/implicits/Readers.scala | 34 +++++++++ ujson/templates/ElemParser.scala | 6 +- 5 files changed, 120 insertions(+), 2 deletions(-) diff --git a/core/src/upickle/core/Visitor.scala b/core/src/upickle/core/Visitor.scala index c8d7d9517..1c5e45347 100644 --- a/core/src/upickle/core/Visitor.scala +++ b/core/src/upickle/core/Visitor.scala @@ -55,6 +55,12 @@ trait Visitor[-T, +J] { * @param index json source position at the start of the number being visited */ def visitFloat64StringParts(s: CharSequence, decIndex: Int, expIndex: Int, index: Int): J + def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int): J = { + visitFloat64StringParts(new String(s, arrOffset, arrLength), decIndex, expIndex, index) + } + def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int): J = { + visitFloat64StringParts(new String(s, arrOffset, arrLength), decIndex, expIndex, index) + } /** * Optional handler for raw double values; can be overriden for performance diff --git a/core/templates/BufferingElemParser.scala b/core/templates/BufferingElemParser.scala index c14fdb85a..67ddc7486 100644 --- a/core/templates/BufferingElemParser.scala +++ b/core/templates/BufferingElemParser.scala @@ -26,6 +26,8 @@ trait BufferingElemParser{ private[this] var buffer: Array[Elem] = null private[this] var bufferGrowCount = 0 private[this] var bufferCopyCount = 0 + def getBuffer = buffer + def getFirstIdx = firstIdx def getBufferGrowCount() = bufferGrowCount def getBufferCopyCount() = bufferCopyCount def getBufferLength() = if (buffer == null) -1 else buffer.length diff --git a/core/templates/ElemUtils.scala b/core/templates/ElemUtils.scala index ca6e671da..9a98ed727 100644 --- a/core/templates/ElemUtils.scala +++ b/core/templates/ElemUtils.scala @@ -103,4 +103,78 @@ object ElemUtils{ i - 1 } + + def parseIntegralNum(arr: Array[Elem], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + val expMul = + if (expIndex == -1) 1 + else { + var mult = 1 + val e = parseLong(arr, arrOffset + expIndex + 1, arrOffset + arrLength) + var i = 0 + while (i < e) { + if (mult >= Long.MaxValue / 10) throw new Abort("expected integer") + mult = mult * 10 + i += 1 + } + mult + } + + val intPortion = { + val end = + if (decIndex != -1) decIndex + else if (expIndex != -1) expIndex + else arrLength + + parseLong(arr, arrOffset, arrOffset + end) * expMul + } + + val decPortion = + if (decIndex == -1) 0 + else { + val end = if (expIndex != -1) expIndex else arrLength + var value = parseLong(arr, arrOffset + decIndex + 1, arrOffset + end) * expMul + var i = end - (decIndex + 1) + while (i > 0) { + value = value / 10 + i -= 1 + } + if (arr(arrOffset) == '-') -value else value + } + + intPortion + decPortion + } + + def parseLong(cs: Array[Elem], start: Int, end: Int): Long = { + + // we store the inverse of the positive sum, to ensure we don't + // incorrectly overflow on Long.MinValue. for positive numbers + // this inverse sum will be inverted before being returned. + var inverseSum: Long = 0L + var inverseSign: Long = -1L + var i: Int = start + + if ((start | end | end - start) < 0) throw new IndexOutOfBoundsException + + if (cs(start) == '-') { + inverseSign = 1L + i += 1 + } + + val size = end - i + if (size <= 0 || size > 19) throw new NumberFormatException(new String(cs, start, end)) + + while (i < end) { + val digit = cs(i).toInt - 48 + if (digit < 0 || 9 < digit) throw new NumberFormatException(new String(cs, start, end)) + inverseSum = inverseSum * 10L - digit + i += 1 + } + + // detect and throw on overflow + if (size == 19 && (inverseSum >= 0 || (inverseSum == Long.MinValue && inverseSign < 0))) { + throw new NumberFormatException(new String(cs, start, end)) + } + + inverseSum * inverseSign + } } \ No newline at end of file diff --git a/implicits/src/upickle/implicits/Readers.scala b/implicits/src/upickle/implicits/Readers.scala index 2606b1d20..45d4f723a 100644 --- a/implicits/src/upickle/implicits/Readers.scala +++ b/implicits/src/upickle/implicits/Readers.scala @@ -80,6 +80,14 @@ trait Readers extends upickle.core.Types ParseUtils.parseIntegralNum(s, decIndex, expIndex, index).toInt } + override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toInt + } + + override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toInt + } + } implicit val FloatReader: Reader[Float] = new NumericReader[Float] { @@ -108,6 +116,15 @@ trait Readers extends upickle.core.Types ParseUtils.parseIntegralNum(s, decIndex, expIndex, index).toShort } + override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toShort + } + + override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toShort + } + + } implicit val ByteReader: Reader[Byte] = new NumericReader[Byte] { @@ -122,6 +139,15 @@ trait Readers extends upickle.core.Types ParseUtils.parseIntegralNum(s, decIndex, expIndex, index).toByte } + override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toByte + } + + override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toByte + } + + } implicit val StringReader: Reader[String] = new SimpleReader[String] { @@ -181,6 +207,14 @@ trait Readers extends upickle.core.Types ParseUtils.parseIntegralNum(s, decIndex, expIndex, index).toLong } + override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toLong + } + + override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toLong + } + } implicit val BigIntReader: Reader[BigInt] = new SimpleStringReader[BigInt]{ diff --git a/ujson/templates/ElemParser.scala b/ujson/templates/ElemParser.scala index 57eb6d737..a32c7f8a1 100644 --- a/ujson/templates/ElemParser.scala +++ b/ujson/templates/ElemParser.scala @@ -167,8 +167,10 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ expIndex: Int, i: Int, j: Int) = { - facade.visitFloat64StringParts( - unsafeNonUnicodeCharSeqForRange(i, j - i), + facade.visitFloat64ElemParts( + getBuffer, + i - getFirstIdx, + j - i, decIndex, expIndex, i From 49b64616a0ed652f4bb0c552dbf26dabf5ce2153 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 21:28:20 +0800 Subject: [PATCH 13/17] ensure visitFloat64ElemParts implementations copy array before parsing, for performance (???) --- core/src/upickle/core/ParseUtils.scala | 71 +------------------ core/templates/ElemUtils.scala | 29 +++++--- implicits/src/upickle/implicits/Readers.scala | 16 ++--- 3 files changed, 31 insertions(+), 85 deletions(-) diff --git a/core/src/upickle/core/ParseUtils.scala b/core/src/upickle/core/ParseUtils.scala index 41ef3f71b..07dc593e3 100644 --- a/core/src/upickle/core/ParseUtils.scala +++ b/core/src/upickle/core/ParseUtils.scala @@ -25,76 +25,11 @@ object ParseUtils { def stringToBytes(s: String) = s.split('-').map(Integer.parseInt(_, 16).toByte) def parseIntegralNum(s: CharSequence, decIndex: Int, expIndex: Int, index: Int) = { - val expMul = - if (expIndex == -1) 1 - else{ - var mult = 1 - val e = parseLong(s, expIndex + 1, s.length()) - var i = 0 - while(i < e){ - if (mult >= Long.MaxValue / 10) throw new Abort("expected integer") - mult = mult * 10 - i += 1 - } - mult - } - - val intPortion = { - val end = - if(decIndex != -1) decIndex - else if (expIndex != -1) expIndex - else s.length - - parseLong(s, 0, end) * expMul - } - - val decPortion = - if (decIndex == -1) 0 - else{ - val end = if(expIndex != -1) expIndex else s.length - var value = parseLong(s, decIndex + 1, end) * expMul - var i = end - (decIndex + 1) - while(i > 0) { - value = value / 10 - i -= 1 - } - if (s.charAt(0) == '-') -value else value - } - - intPortion + decPortion + val arr = s.toString.toCharArray + CharUtils.parseIntegralNum(arr, 0, arr.length, decIndex, expIndex) } def parseLong(cs: CharSequence, start: Int, end: Int): Long = { - - // we store the inverse of the positive sum, to ensure we don't - // incorrectly overflow on Long.MinValue. for positive numbers - // this inverse sum will be inverted before being returned. - var inverseSum: Long = 0L - var inverseSign: Long = -1L - var i: Int = start - - if ((start | end | end - start | (cs.length - end)) < 0) throw new IndexOutOfBoundsException - - if (cs.charAt(start) == '-') { - inverseSign = 1L - i += 1 - } - - val size = end - i - if (size <= 0 || size > 19) throw new NumberFormatException(cs.toString.substring(start, end)) - - while (i < end) { - val digit = cs.charAt(i).toInt - 48 - if (digit < 0 || 9 < digit) throw new NumberFormatException(cs.toString.substring(start, end)) - inverseSum = inverseSum * 10L - digit - i += 1 - } - - // detect and throw on overflow - if (size == 19 && (inverseSum >= 0 || (inverseSum == Long.MinValue && inverseSign < 0))) { - throw new NumberFormatException(cs.toString.substring(start, end)) - } - - inverseSum * inverseSign + CharUtils.parseLong(cs.toString.toCharArray, start, end) } def reject(j: Int): PartialFunction[Throwable, Nothing] = { diff --git a/core/templates/ElemUtils.scala b/core/templates/ElemUtils.scala index 9a98ed727..421791723 100644 --- a/core/templates/ElemUtils.scala +++ b/core/templates/ElemUtils.scala @@ -104,7 +104,7 @@ object ElemUtils{ } - def parseIntegralNum(arr: Array[Elem], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { + def parseIntegralNum(arr: Array[Elem], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int) = { val expMul = if (expIndex == -1) 1 else { @@ -144,35 +144,46 @@ object ElemUtils{ intPortion + decPortion } - def parseLong(cs: Array[Elem], start: Int, end: Int): Long = { + def parseLong(cs0: Array[Elem], start0: Int, end0: Int): Long = { + // If we do not copy the data from `cs0` into our own local array before + // parsing it, we take a significant performance penalty in the + // `integers Read` benchmarks, but *only* when run together with the rest + // of the benchmarks! When `integers Read` isrun alone, this does not + // happen, and is presumably something to do with the JIT compiler. + + // Since any real world use case would exercise all sorts of code paths, + // it would more closely resemble the "all benchmarks together" case + // rather, and so we leave this copy in-place to optimize performance + // for that scenario. + val cs = new Array[Elem](end0 - start0) + System.arraycopy(cs0, start0, cs, 0, end0 - start0) // we store the inverse of the positive sum, to ensure we don't // incorrectly overflow on Long.MinValue. for positive numbers // this inverse sum will be inverted before being returned. var inverseSum: Long = 0L var inverseSign: Long = -1L - var i: Int = start - - if ((start | end | end - start) < 0) throw new IndexOutOfBoundsException + var i: Int = 0 + val end = end0 - start0 - if (cs(start) == '-') { + if (cs(0) == '-') { inverseSign = 1L i += 1 } val size = end - i - if (size <= 0 || size > 19) throw new NumberFormatException(new String(cs, start, end)) + if (size <= 0 || size > 19) throw new NumberFormatException(new String(cs)) while (i < end) { val digit = cs(i).toInt - 48 - if (digit < 0 || 9 < digit) throw new NumberFormatException(new String(cs, start, end)) + if (digit < 0 || 9 < digit) throw new NumberFormatException(new String(cs)) inverseSum = inverseSum * 10L - digit i += 1 } // detect and throw on overflow if (size == 19 && (inverseSum >= 0 || (inverseSum == Long.MinValue && inverseSign < 0))) { - throw new NumberFormatException(new String(cs, start, end)) + throw new NumberFormatException(new String(cs)) } inverseSum * inverseSign diff --git a/implicits/src/upickle/implicits/Readers.scala b/implicits/src/upickle/implicits/Readers.scala index 45d4f723a..b68a47241 100644 --- a/implicits/src/upickle/implicits/Readers.scala +++ b/implicits/src/upickle/implicits/Readers.scala @@ -81,11 +81,11 @@ trait Readers extends upickle.core.Types } override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toInt + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toInt } override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toInt + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toInt } } @@ -117,11 +117,11 @@ trait Readers extends upickle.core.Types } override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toShort + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toShort } override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toShort + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toShort } @@ -140,11 +140,11 @@ trait Readers extends upickle.core.Types } override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toByte + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toByte } override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toByte + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toByte } @@ -208,11 +208,11 @@ trait Readers extends upickle.core.Types } override def visitFloat64ByteParts(s: Array[Byte], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toLong + ByteUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toLong } override def visitFloat64CharParts(s: Array[Char], arrOffset: Int, arrLength: Int, decIndex: Int, expIndex: Int, index: Int) = { - CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex, index).toLong + CharUtils.parseIntegralNum(s, arrOffset, arrLength, decIndex, expIndex).toLong } } From 66d9af8634fd3d837fb051ecc41f7cf9669b5b72 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 21:58:23 +0800 Subject: [PATCH 14/17] cleanup --- core/templates/BufferingElemParser.scala | 3 --- core/templates/WrapElemArrayCharSeq.scala | 29 ----------------------- 2 files changed, 32 deletions(-) diff --git a/core/templates/BufferingElemParser.scala b/core/templates/BufferingElemParser.scala index 67ddc7486..8fbe08093 100644 --- a/core/templates/BufferingElemParser.scala +++ b/core/templates/BufferingElemParser.scala @@ -160,9 +160,6 @@ trait BufferingElemParser{ def unsafeCharSeqForRange(start: Int, length: Int) = { new WrapElemArrayCharSeq(buffer, start - firstIdx, length) } - def unsafeNonUnicodeCharSeqForRange(start: Int, length: Int) = { - new WrapElemArrayCharSeq.NonUnicode(buffer, start - firstIdx, length) - } def appendElemsToBuilder(elems: ElemBuilder, elemsStart: Int, elemsLength: Int) = { elems.appendAll(buffer, elemsStart - firstIdx, elemsLength) } diff --git a/core/templates/WrapElemArrayCharSeq.scala b/core/templates/WrapElemArrayCharSeq.scala index c162ce1bb..3bd58ae75 100644 --- a/core/templates/WrapElemArrayCharSeq.scala +++ b/core/templates/WrapElemArrayCharSeq.scala @@ -32,32 +32,3 @@ class WrapElemArrayCharSeq(arr: Array[Elem], start: Int, length0: Int) extends C } } } - -object WrapElemArrayCharSeq{ - class NonUnicode(arr: Array[Elem], start: Int, length0: Int) extends CharSequence { - // assert( - // start < arr.length && start + length0 <= arr.length && start >= 0 && length0 >= 0, - // s"WrapElemArrayCharSeq(${arr.length}, $start, $length0)" - // ) - def length() = length0 - - def charAt(index: Int) = { - arr(index + start).toChar - } - - def subSequence(newStart: Int, newEnd: Int) = { - new WrapElemArrayCharSeq(arr, start + newStart, newEnd - newStart) - } - - private[this] var toString0: String = null - - override def toString = { - if (toString0 != null) toString0 - else { - val res = ElemOps.newString(arr, start, length0) - toString0 = res - res - } - } - } -} \ No newline at end of file From 05f8b98634bb4e6b19d3047f290a5c430052b294 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 Mar 2023 23:15:22 +0800 Subject: [PATCH 15/17] add back bounds check to parseLong --- core/templates/ElemUtils.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/templates/ElemUtils.scala b/core/templates/ElemUtils.scala index 421791723..59647b5c6 100644 --- a/core/templates/ElemUtils.scala +++ b/core/templates/ElemUtils.scala @@ -145,6 +145,8 @@ object ElemUtils{ } def parseLong(cs0: Array[Elem], start0: Int, end0: Int): Long = { + if ((start0 | end0 | end0 - start0 | (cs0.length - end0)) < 0) throw new IndexOutOfBoundsException + // If we do not copy the data from `cs0` into our own local array before // parsing it, we take a significant performance penalty in the // `integers Read` benchmarks, but *only* when run together with the rest From 4e1c08bb4d12202052322488a833ab6c47c00fee Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Mon, 20 Mar 2023 00:37:53 +0800 Subject: [PATCH 16/17] fix scala 3 compile --- implicits/src-3/upickle/implicits/Writers.scala | 2 +- implicits/src-3/upickle/implicits/macros.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/implicits/src-3/upickle/implicits/Writers.scala b/implicits/src-3/upickle/implicits/Writers.scala index d14f4a7af..af9fa475e 100644 --- a/implicits/src-3/upickle/implicits/Writers.scala +++ b/implicits/src-3/upickle/implicits/Writers.scala @@ -19,7 +19,7 @@ trait WritersVersionSpecific def writer = new CaseClassWriter[T] { def length(v: T) = macros.writeLength[T](outerThis, v) - override def write0[R](out: Visitor[_, R], v: V): R = { + override def write0[R](out: Visitor[_, R], v: T): R = { if (v == null) out.visitNull(-1) else { val ctx = out.visitObject(length(v), true, -1) diff --git a/implicits/src-3/upickle/implicits/macros.scala b/implicits/src-3/upickle/implicits/macros.scala index adb5224e9..5b7760f8b 100644 --- a/implicits/src-3/upickle/implicits/macros.scala +++ b/implicits/src-3/upickle/implicits/macros.scala @@ -143,7 +143,7 @@ def writeSnippetsImpl[R, T, WS <: Tuple](thisOuter: Expr[upickle.core.Types with case '[IsInt[index]] => val select = Select.unique(v.asTerm, rawLabel.name).asExprOf[Any] val snippet = '{ - ${self}.writeSnippet[R, tpe]( + ${self}.writeSnippetMappedName[R, tpe]( ${ctx}, ${thisOuter}.objectAttributeKeyWriteMap(${Expr(label)}), summonInline[Tuple.Elem[WS, index]], From 8aec3b754380062dcabcd35657a304564947323f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Mon, 20 Mar 2023 09:04:24 +0800 Subject: [PATCH 17/17] mima --- ujson/templates/ElemParser.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ujson/templates/ElemParser.scala b/ujson/templates/ElemParser.scala index a32c7f8a1..b3c5b1034 100644 --- a/ujson/templates/ElemParser.scala +++ b/ujson/templates/ElemParser.scala @@ -529,7 +529,8 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ * This method expects the data to be in UTF-16 and accesses it as * chars. */ - protected[this] final def parseStringSimple(i: Int, safeIndex0: Int = 0): Int = { + protected[this] final def parseStringSimple(i: Int): Int = parseStringSimple(i, 0) + protected[this] final def parseStringSimple(i: Int, safeIndex0: Int): Int = { var j = i var safeIndex = safeIndex0 while (true) { @@ -554,7 +555,8 @@ abstract class ElemParser[J] extends upickle.core.BufferingElemParser{ /** * Parse a string that is known to have escape sequences. */ - protected[this] final def parseStringComplex(i0: Int, safeIndex0: Int = 0): Int = { + protected[this] final def parseStringComplex(i0: Int): Int = parseStringComplex(i0, 0) + protected[this] final def parseStringComplex(i0: Int, safeIndex0: Int): Int = { val elemOps = upickle.core.ElemOps var i = i0 var safeIndex = safeIndex0