chore: build data section tree from map file

DarkAtra · Jun 27, 2023 · 7e7e260 · 7e7e260
1 parent f80537e
commit 7e7e260
Show file tree

Hide file tree

Showing 35 changed files with 296 additions and 133 deletions.
diff --git a/...2/map/serialization/MapFileCompression.kt → .../darkatra/bfme2/map/MapFileCompression.kt b/...2/map/serialization/MapFileCompression.kt → .../darkatra/bfme2/map/MapFileCompression.kt
@@ -1,4 +1,4 @@
-package de.darkatra.bfme2.map.serialization
+package de.darkatra.bfme2.map
 
 enum class MapFileCompression(
     internal val fourCC: String

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/riverarea/RiverArea.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/riverarea/RiverArea.kt
@@ -2,7 +2,7 @@ package de.darkatra.bfme2.map.riverarea
 
 import de.darkatra.bfme2.Color
 import de.darkatra.bfme2.Line2D
-import de.darkatra.bfme2.map.serialization.ByteColorSerde
+import de.darkatra.bfme2.map.serialization.FourByteColorSerde
 import de.darkatra.bfme2.map.serialization.Serialize
 
 data class RiverArea(
@@ -15,7 +15,7 @@ data class RiverArea(
     val noiseTexture: String,
     val alphaEdgeTexture: String,
     val sparkleTexture: String,
-    val color: @Serialize(using = ByteColorSerde::class) Color,
+    val color: @Serialize(using = FourByteColorSerde::class) Color,
     val alpha: Float,
     val waterHeight: UInt,
     val minimumWaterLod: String,

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/AssetListSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/AssetListSerde.kt
@@ -1,21 +1,25 @@
 package de.darkatra.bfme2.map.serialization
 
 import com.google.common.io.CountingInputStream
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionHolder
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import java.io.OutputStream
 
-internal class AssetListSerde<T>(
+internal class AssetListSerde<T : Any>(
     private val serializationContext: SerializationContext,
     private val entrySerde: Serde<T>,
     private val preProcessor: PreProcessor<List<T>>,
     private val postProcessor: PostProcessor<List<T>>
 ) : Serde<List<T>> {
 
-    override fun calculateByteCount(data: List<T>): Long {
-        return data.sumOf {
-            4 + 2 + 4 + entrySerde.calculateByteCount(it)
-        }
+    override fun collectDataSections(data: List<T>): DataSection {
+        return DataSectionHolder(
+            containingData = data.map {
+                entrySerde.collectDataSections(it)
+            }
+        )
     }
 
     override fun serialize(outputStream: OutputStream, data: List<T>) {

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/BlendCountSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/BlendCountSerde.kt
@@ -1,6 +1,7 @@
 package de.darkatra.bfme2.map.serialization
 
 import com.google.common.io.CountingInputStream
+import de.darkatra.bfme2.map.serialization.model.DataSection
 import de.darkatra.bfme2.map.serialization.postprocessing.NoopPostProcessor
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.NoopPreProcessor
@@ -15,7 +16,9 @@ internal class BlendCountSerde(
 
     private val uIntSerde = UIntSerde(context, NoopPreProcessor(), NoopPostProcessor())
 
-    override fun calculateByteCount(data: UInt): Long = uIntSerde.calculateByteCount(data)
+    override fun collectDataSections(data: UInt): DataSection {
+        return uIntSerde.collectDataSections(data)
+    }
 
     override fun serialize(outputStream: OutputStream, data: UInt) {
         uIntSerde.serialize(

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/BooleanSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/BooleanSerde.kt
@@ -1,5 +1,7 @@
 package de.darkatra.bfme2.map.serialization
 
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import de.darkatra.bfme2.readBoolean
@@ -19,5 +21,5 @@ internal class BooleanSerde(
     postProcessor
 ) {
 
-    override fun calculateByteCount(data: Boolean): Long = 1
+    override fun collectDataSections(data: Boolean): DataSection = DataSectionLeaf.BOOLEAN
 }
diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ByteSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ByteSerde.kt
@@ -1,5 +1,7 @@
 package de.darkatra.bfme2.map.serialization
 
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import de.darkatra.bfme2.readByte
@@ -19,5 +21,5 @@ internal class ByteSerde(
     postProcessor
 ) {
 
-    override fun calculateByteCount(data: Byte): Long = 1
+    override fun collectDataSections(data: Byte): DataSection = DataSectionLeaf.BYTE
 }
diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ColorSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ColorSerde.kt
@@ -2,10 +2,11 @@ package de.darkatra.bfme2.map.serialization
 
 import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.Color
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.postprocessing.NoopPostProcessor
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
+import de.darkatra.bfme2.map.serialization.preprocessing.NoopPreProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
-import de.darkatra.bfme2.readUInt
-import de.darkatra.bfme2.writeUInt
 import java.io.OutputStream
 
 internal class ColorSerde(
@@ -14,17 +15,19 @@ internal class ColorSerde(
     private val postProcessor: PostProcessor<Color>
 ) : Serde<Color> {
 
-    override fun calculateByteCount(data: Color): Long = 4
+    private val uIntSerde = UIntSerde(context, NoopPreProcessor(), NoopPostProcessor())
+
+    override fun collectDataSections(data: Color): DataSection = uIntSerde.collectDataSections(data.rgba)
 
     override fun serialize(outputStream: OutputStream, data: Color) {
         preProcessor.preProcess(data, context).let {
-            outputStream.writeUInt(it.rgba)
+            uIntSerde.serialize(outputStream, it.rgba)
         }
     }
 
     override fun deserialize(inputStream: CountingInputStream): Color {
         return Color(
-            rgba = inputStream.readUInt()
+            rgba = uIntSerde.deserialize(inputStream)
         ).also {
             postProcessor.postProcess(it, context)
         }

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ConditionalSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ConditionalSerde.kt
@@ -3,6 +3,7 @@ package de.darkatra.bfme2.map.serialization
 import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.InvalidDataException
 import de.darkatra.bfme2.map.Asset
+import de.darkatra.bfme2.map.serialization.model.DataSection
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import java.io.OutputStream
@@ -37,12 +38,12 @@ internal class ConditionalSerde(
         Pair(assetName.name, serdeFactory.getSerde(assetType))
     }
 
-    override fun calculateByteCount(data: Any): Long {
-        // TODO: validate that the serializationContext is updated during byte count calculation and peek() behaves as expected
-        val assetName = serializationContext.peek().assetName
-        val serde = serdes[assetName]
-            ?: throw IllegalStateException("Could not find serde for '$assetName' calculating byte count for $currentElementName. Expected one of: ${serdes.keys}")
-        return serde.calculateByteCount(data)
+    override fun collectDataSections(data: Any): DataSection {
+        val asset = data::class.findAnnotation<Asset>()
+            ?: throw IllegalStateException("Could not find asset annotation for $currentElementName. Expected one of: ${serdes.keys}")
+        val serde = serdes[asset.name]
+            ?: throw IllegalStateException("Could not find serde for '${asset.name}' calculating byte count for $currentElementName. Expected one of: ${serdes.keys}")
+        return serde.collectDataSections(data)
     }
 
     override fun serialize(outputStream: OutputStream, data: Any) {

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/EnumSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/EnumSerde.kt
@@ -2,6 +2,7 @@ package de.darkatra.bfme2.map.serialization
 
 import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.ConversionException
+import de.darkatra.bfme2.map.serialization.model.DataSection
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import java.io.OutputStream
@@ -33,8 +34,8 @@ internal class EnumSerde<T : Enum<*>>(
         }
     }
 
-    override fun calculateByteCount(data: T): Long {
-        return serde.calculateByteCount(enumValueGetter.call(data)!!)
+    override fun collectDataSections(data: T): DataSection {
+        return serde.collectDataSections(enumValueGetter.call(data)!!)
     }
 
     override fun serialize(outputStream: OutputStream, data: T) {

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/FloatSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/FloatSerde.kt
@@ -1,5 +1,7 @@
 package de.darkatra.bfme2.map.serialization
 
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import de.darkatra.bfme2.readFloat
@@ -19,5 +21,5 @@ internal class FloatSerde(
     postProcessor
 ) {
 
-    override fun calculateByteCount(data: Float): Long = 4
+    override fun collectDataSections(data: Float): DataSection = DataSectionLeaf.FLOAT
 }
diff --git a/...bfme2/map/serialization/ByteColorSerde.kt → ...2/map/serialization/FourByteColorSerde.kt b/...bfme2/map/serialization/ByteColorSerde.kt → ...2/map/serialization/FourByteColorSerde.kt
@@ -3,19 +3,21 @@ package de.darkatra.bfme2.map.serialization
 import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.Color
 import de.darkatra.bfme2.InvalidDataException
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import de.darkatra.bfme2.readByte
 import de.darkatra.bfme2.writeByte
 import java.io.OutputStream
 
-internal class ByteColorSerde(
+internal class FourByteColorSerde(
     private val context: SerializationContext,
     private val preProcessor: PreProcessor<Color>,
     private val postProcessor: PostProcessor<Color>
 ) : Serde<Color> {
 
-    override fun calculateByteCount(data: Color): Long = 4
+    override fun collectDataSections(data: Color): DataSection = DataSectionLeaf(4)
 
     override fun serialize(outputStream: OutputStream, data: Color) {
         preProcessor.preProcess(data, context).let {
@@ -34,7 +36,7 @@ internal class ByteColorSerde(
 
         val unusedAlpha = inputStream.readByte()
         if (unusedAlpha != 0.toByte()) {
-            throw InvalidDataException("Expected unusedAlpha to be 0 using ${ByteColorSerde::class.simpleName}.")
+            throw InvalidDataException("Expected unusedAlpha to be 0 using ${FourByteColorSerde::class.simpleName}.")
         }
 
         return Color(

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/FourByteStringSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/FourByteStringSerde.kt
@@ -2,6 +2,8 @@ package de.darkatra.bfme2.map.serialization
 
 import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.InvalidDataException
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import java.io.OutputStream
@@ -13,7 +15,7 @@ internal class FourByteStringSerde(
     private val postProcessor: PostProcessor<String>
 ) : Serde<String> {
 
-    override fun calculateByteCount(data: String): Long = 4
+    override fun collectDataSections(data: String): DataSection = DataSectionLeaf(4)
 
     override fun serialize(outputStream: OutputStream, data: String) {
         preProcessor.preProcess(data, context).let {

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/HeightMapDependentMapSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/HeightMapDependentMapSerde.kt
@@ -4,6 +4,9 @@ import com.google.common.collect.HashBasedTable
 import com.google.common.collect.Table
 import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.map.heightmap.HeightMap
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionHolder
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import de.darkatra.bfme2.readByte
@@ -36,22 +39,25 @@ internal class HeightMapDependentMapSerde<V : Any>(
         SAGE_BOOLEAN
     }
 
-    override fun calculateByteCount(data: Table<UInt, UInt, V>): Long {
+    override fun collectDataSections(data: Table<UInt, UInt, V>): DataSection {
 
         val width = data.rowKeySet().size.toUInt()
         val height = data.columnKeySet().size.toUInt()
 
-        return (0u until width step 1).sumOf { x ->
-            (0u until height step 1).sumOf { y ->
-                when (mode) {
-                    Mode.DEFAULT -> valueSerde.calculateByteCount(data[x, y]!!)
-                    else -> when (x % 8u == 0u) {
-                        true -> 1
-                        else -> 0
+        return DataSectionHolder(
+            containingData = (0u until width step 1).map { x ->
+                DataSectionHolder(
+                    containingData = buildList {
+                        (0u until height step 1).forEach { y ->
+                            when {
+                                mode == Mode.DEFAULT -> add(valueSerde.collectDataSections(data[x, y]!!))
+                                x % 8u == 0u -> add(DataSectionLeaf(1))
+                            }
+                        }
                     }
-                }
+                )
             }
-        }
+        )
     }
 
     override fun serialize(outputStream: OutputStream, data: Table<UInt, UInt, V>) {

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/IntSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/IntSerde.kt
@@ -1,5 +1,7 @@
 package de.darkatra.bfme2.map.serialization
 
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import de.darkatra.bfme2.readInt
@@ -19,5 +21,5 @@ internal class IntSerde(
     postProcessor
 ) {
 
-    override fun calculateByteCount(data: Int): Long = 4
+    override fun collectDataSections(data: Int): DataSection = DataSectionLeaf.INT
 }
diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ListSerde.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ListSerde.kt
@@ -1,6 +1,9 @@
 package de.darkatra.bfme2.map.serialization
 
 import com.google.common.io.CountingInputStream
+import de.darkatra.bfme2.map.serialization.model.DataSection
+import de.darkatra.bfme2.map.serialization.model.DataSectionHolder
+import de.darkatra.bfme2.map.serialization.model.DataSectionLeaf
 import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
 import de.darkatra.bfme2.map.serialization.preprocessing.PreProcessor
 import de.darkatra.bfme2.readByte
@@ -57,18 +60,21 @@ internal class ListSerde<T>(
         BYTE
     }
 
-    override fun calculateByteCount(data: List<T>): Long {
-        return when (mode) {
-            Mode.DEFAULT -> when (sizeType) {
-                SizeType.UINT -> 4
-                SizeType.USHORT -> 2
-                SizeType.BYTE -> 1
+    override fun collectDataSections(data: List<T>): DataSection {
+        return DataSectionHolder(
+            containingData = buildList {
+                if (mode == Mode.DEFAULT) {
+                    add(
+                        when (sizeType) {
+                            SizeType.UINT -> DataSectionLeaf.INT
+                            SizeType.USHORT -> DataSectionLeaf.SHORT
+                            SizeType.BYTE -> DataSectionLeaf.BYTE
+                        }
+                    )
+                }
+                addAll(data.map { entrySerde.collectDataSections(it) })
             }
-
-            else -> 0
-        } + data.sumOf {
-            entrySerde.calculateByteCount(it)
-        }
+        )
     }
 
     override fun serialize(outputStream: OutputStream, data: List<T>) {

diff --git a/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/MapFileReader.kt b/map/src/main/kotlin/de/darkatra/bfme2/map/serialization/MapFileReader.kt
@@ -5,6 +5,8 @@ import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.InvalidDataException
 import de.darkatra.bfme2.SkippingInputStream
 import de.darkatra.bfme2.map.MapFile
+import de.darkatra.bfme2.map.MapFileCompression
+import de.darkatra.bfme2.map.serialization.model.AssetEntry
 import de.darkatra.bfme2.read7BitIntPrefixedString
 import de.darkatra.bfme2.readUInt
 import de.darkatra.bfme2.readUShort
@@ -46,7 +48,7 @@ class MapFileReader {
                 serializationContext.pop()
 
                 val currentEndPosition = inputStream.count
-                val expectedEndPosition = currentAsset.endPosition
+                val expectedEndPosition = serializationContext.currentEndPosition
                 if (!serializationContext.debugMode && currentEndPosition != expectedEndPosition) {
                     throw InvalidDataException("Error reading '${currentAsset.assetName}'. Expected reader to be at position $expectedEndPosition, but was at $currentEndPosition.")
                 }