chore: WIP MapFileWriter

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/AssetListSerde.kt

@@ -14,10 +14,10 @@ internal class AssetListSerde<T : Any>(
     private val postProcessor: PostProcessor<List<T>>
 ) : Serde<List<T>> {
 
-    override fun collectDataSections(data: List<T>): DataSection {
+    override fun calculateDataSection(data: List<T>): DataSection {
         return DataSectionHolder(
             containingData = data.map {
-                entrySerde.collectDataSections(it)
+                entrySerde.calculateDataSection(it)
             }
         )
     }
@@ -26,7 +26,7 @@ internal class AssetListSerde<T : Any>(
 
         preProcessor.preProcess(data, serializationContext).let { list ->
             list.forEach { entry ->
-                // TODO: serialize asset header
+                MapFileWriter.writeAsset(outputStream, serializationContext, entry)
                 entrySerde.serialize(outputStream, entry)
             }
         }

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/BlendCountSerde.kt

@@ -16,8 +16,8 @@ internal class BlendCountSerde(
 
     private val uIntSerde = UIntSerde(context, NoopPreProcessor(), NoopPostProcessor())
 
-    override fun collectDataSections(data: UInt): DataSection {
-        return uIntSerde.collectDataSections(data)
+    override fun calculateDataSection(data: UInt): DataSection {
+        return uIntSerde.calculateDataSection(data)
     }
 
     override fun serialize(outputStream: OutputStream, data: UInt) {

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/BooleanSerde.kt

@@ -21,5 +21,5 @@ internal class BooleanSerde(
     postProcessor
 ) {
 
-    override fun collectDataSections(data: Boolean): DataSection = DataSectionLeaf.BOOLEAN
+    override fun calculateDataSection(data: Boolean): DataSection = DataSectionLeaf.BOOLEAN
 }

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ByteSerde.kt

@@ -21,5 +21,5 @@ internal class ByteSerde(
     postProcessor
 ) {
 
-    override fun collectDataSections(data: Byte): DataSection = DataSectionLeaf.BYTE
+    override fun calculateDataSection(data: Byte): DataSection = DataSectionLeaf.BYTE
 }

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ColorSerde.kt

@@ -17,7 +17,7 @@ internal class ColorSerde(
 
     private val uIntSerde = UIntSerde(context, NoopPreProcessor(), NoopPostProcessor())
 
-    override fun collectDataSections(data: Color): DataSection = uIntSerde.collectDataSections(data.rgba)
+    override fun calculateDataSection(data: Color): DataSection = uIntSerde.calculateDataSection(data.rgba)
 
     override fun serialize(outputStream: OutputStream, data: Color) {
         preProcessor.preProcess(data, context).let {

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ConditionalSerde.kt

@@ -38,12 +38,12 @@ internal class ConditionalSerde(
         Pair(assetName.name, serdeFactory.getSerde(assetType))
     }
 
-    override fun collectDataSections(data: Any): DataSection {
+    override fun calculateDataSection(data: Any): DataSection {
         val asset = data::class.findAnnotation<Asset>()
             ?: throw IllegalStateException("Could not find asset annotation for $currentElementName. Expected one of: ${serdes.keys}")
         val serde = serdes[asset.name]
             ?: throw IllegalStateException("Could not find serde for '${asset.name}' calculating byte count for $currentElementName. Expected one of: ${serdes.keys}")
-        return serde.collectDataSections(data)
+        return serde.calculateDataSection(data)
     }
 
     override fun serialize(outputStream: OutputStream, data: Any) {

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/EnumSerde.kt

@@ -34,8 +34,8 @@ internal class EnumSerde<T : Enum<*>>(
         }
     }
 
-    override fun collectDataSections(data: T): DataSection {
-        return serde.collectDataSections(enumValueGetter.call(data)!!)
+    override fun calculateDataSection(data: T): DataSection {
+        return serde.calculateDataSection(enumValueGetter.call(data)!!)
     }
 
     override fun serialize(outputStream: OutputStream, data: T) {

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/FloatSerde.kt

@@ -21,5 +21,5 @@ internal class FloatSerde(
     postProcessor
 ) {
 
-    override fun collectDataSections(data: Float): DataSection = DataSectionLeaf.FLOAT
+    override fun calculateDataSection(data: Float): DataSection = DataSectionLeaf.FLOAT
 }

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/FourByteColorSerde.kt

@@ -17,7 +17,7 @@ internal class FourByteColorSerde(
     private val postProcessor: PostProcessor<Color>
 ) : Serde<Color> {
 
-    override fun collectDataSections(data: Color): DataSection = DataSectionLeaf(4)
+    override fun calculateDataSection(data: Color): DataSection = DataSectionLeaf(4)
 
     override fun serialize(outputStream: OutputStream, data: Color) {
         preProcessor.preProcess(data, context).let {

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/FourByteStringSerde.kt

@@ -15,7 +15,7 @@ internal class FourByteStringSerde(
     private val postProcessor: PostProcessor<String>
 ) : Serde<String> {
 
-    override fun collectDataSections(data: String): DataSection = DataSectionLeaf(4)
+    override fun calculateDataSection(data: String): DataSection = DataSectionLeaf(4)
 
     override fun serialize(outputStream: OutputStream, data: String) {
         preProcessor.preProcess(data, context).let {

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/HeightMapDependentMapSerde.kt

@@ -39,7 +39,7 @@ internal class HeightMapDependentMapSerde<V : Any>(
         SAGE_BOOLEAN
     }
 
-    override fun collectDataSections(data: Table<UInt, UInt, V>): DataSection {
+    override fun calculateDataSection(data: Table<UInt, UInt, V>): DataSection {
 
         val width = data.rowKeySet().size.toUInt()
         val height = data.columnKeySet().size.toUInt()
@@ -50,7 +50,7 @@ internal class HeightMapDependentMapSerde<V : Any>(
                     containingData = buildList {
                         (0u until height step 1).forEach { y ->
                             when {
-                                mode == Mode.DEFAULT -> add(valueSerde.collectDataSections(data[x, y]!!))
+                                mode == Mode.DEFAULT -> add(valueSerde.calculateDataSection(data[x, y]!!))
                                 x % 8u == 0u -> add(DataSectionLeaf(1))
                             }
                         }
@@ -73,8 +73,8 @@ internal class HeightMapDependentMapSerde<V : Any>(
                     when (mode) {
                         Mode.DEFAULT -> valueSerde.serialize(outputStream, map[x, y]!!)
                         else -> {
-                            // FIXME: this if statement does look weird
-                            if (x > 0u && x % 8u == 0u) {
+                            // TODO: check if the if statement logic is correct
+                            if (x % 8u == 0u) {
                                 outputStream.writeByte(temp)
                             }
                             temp = temp or ((if (map[x, y] as Boolean) 1 else 0) shl (x % 8u).toInt()).toByte()

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/IntSerde.kt

@@ -21,5 +21,5 @@ internal class IntSerde(
     postProcessor
 ) {
 
-    override fun collectDataSections(data: Int): DataSection = DataSectionLeaf.INT
+    override fun calculateDataSection(data: Int): DataSection = DataSectionLeaf.INT
 }

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/ListSerde.kt

@@ -16,6 +16,7 @@ import java.io.OutputStream
 
 @UseSerdeProperties(ListSerde.Properties::class)
 internal class ListSerde<T>(
+    annotationProcessingContext: AnnotationProcessingContext,
     private val context: SerializationContext,
     private val entrySerde: Serde<T>,
     private val preProcessor: PreProcessor<List<T>>,
@@ -27,6 +28,8 @@ internal class ListSerde<T>(
     private val sharedDataKey: String
 ) : Serde<List<T>> {
 
+    private val currentElementName = annotationProcessingContext.getCurrentElement().getName()
+
     init {
         if (mode == Mode.FIXED && size == 0u) {
             error("${ListSerde::class.simpleName} requires 'size' to be set via ${Properties::class.qualifiedName} when 'mode' is '${Mode.FIXED}'.")
@@ -60,7 +63,7 @@ internal class ListSerde<T>(
         BYTE
     }
 
-    override fun collectDataSections(data: List<T>): DataSection {
+    override fun calculateDataSection(data: List<T>): DataSection {
         return DataSectionHolder(
             containingData = buildList {
                 if (mode == Mode.DEFAULT) {
@@ -72,7 +75,7 @@ internal class ListSerde<T>(
                         }
                     )
                 }
-                addAll(data.map { entrySerde.collectDataSections(it) })
+                addAll(data.map { entrySerde.calculateDataSection(it) })
             }
         )
     }
@@ -82,20 +85,12 @@ internal class ListSerde<T>(
         preProcessor.preProcess(data, context).let { list ->
 
             val numberOfListEntries = list.size.toUInt()
-            when (mode) {
-                Mode.DEFAULT -> when (sizeType) {
+            if (mode == Mode.DEFAULT) {
+                when (sizeType) {
                     SizeType.UINT -> outputStream.writeUInt(numberOfListEntries)
                     SizeType.USHORT -> outputStream.writeUShort(numberOfListEntries.toUShort())
                     SizeType.BYTE -> outputStream.writeByte(numberOfListEntries.toByte())
                 }
-
-                Mode.FIXED -> if (numberOfListEntries != size) {
-                    throw IllegalStateException("Expected '' to have size '$size'.")
-                }
-
-                Mode.SHARED_DATA -> if (numberOfListEntries != size) {
-                    throw IllegalStateException("Expected '' to have size '${context.sharedData[sharedDataKey]}'.")
-                }
             }
 
             list.forEach { entry ->

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/MapFileSerde.kt

@@ -3,9 +3,7 @@ package de.darkatra.bfme2.map.serialization
 import com.google.common.io.CountingInputStream
 import de.darkatra.bfme2.map.Asset
 import de.darkatra.bfme2.map.MapFile
-import de.darkatra.bfme2.map.serialization.model.DataSection
-import de.darkatra.bfme2.map.serialization.model.MapFileDataSectionHolder
-import de.darkatra.bfme2.map.serialization.postprocessing.PostProcessor
+import de.darkatra.bfme2.map.serialization.model.DataSectionHolder
 import de.darkatra.bfme2.map.toKClass
 import java.io.OutputStream
 import kotlin.reflect.KProperty
@@ -18,37 +16,54 @@ import kotlin.time.measureTime
 
 internal class MapFileSerde(
     private val serializationContext: SerializationContext,
-    private val serdes: List<Serde<*>>,
-    private val postProcessor: PostProcessor<MapFile>
+    private val serdes: List<Serde<*>>
 ) : Serde<MapFile> {
 
     private val primaryConstructor = MapFile::class.primaryConstructor
         ?: error("${MapFile::class.simpleName} is required to have a primary constructor.")
 
     private val parameters = primaryConstructor.valueParameters
+    private val parameterToField = parameters.associateWith { parameter ->
+        val fieldForParameter = MapFile::class.members
+            .filterIsInstance<KProperty<*>>()
+            .find { field -> field.name == parameter.name }!!
+        if (fieldForParameter.getter.visibility != KVisibility.PUBLIC && fieldForParameter.getter.visibility != KVisibility.INTERNAL) {
+            throw IllegalStateException("Field for parameter '${parameter.name}' is not public or internal.")
+        }
+        fieldForParameter
+    }
 
-    override fun collectDataSections(data: MapFile): DataSection {
-
-        return MapFileDataSectionHolder(
-            containingData = parameters.mapIndexed { index, parameter ->
-                val fieldForParameter = MapFile::class.members
-                    .filterIsInstance<KProperty<*>>()
-                    .first { field -> field.name == parameter.name }
+    override fun calculateDataSection(data: MapFile): DataSectionHolder {
 
-                if (fieldForParameter.getter.visibility == KVisibility.PUBLIC || fieldForParameter.getter.visibility == KVisibility.INTERNAL) {
-                    @Suppress("UNCHECKED_CAST")
-                    val serde = serdes[index] as Serde<Any>
-                    val fieldData = fieldForParameter.getter.call(data)!!
-                    serde.collectDataSections(fieldData)
-                } else {
-                    throw IllegalStateException("Could not collect data sections for parameter '${parameter.name}' because it's getter is not public or internal.")
-                }
-            }
+        return DataSectionHolder(
+            containingData = parameterToField.entries.mapIndexed { index, (p, fieldForParameter) ->
+                @Suppress("UNCHECKED_CAST")
+                val serde = serdes[index] as Serde<Any>
+                val fieldData = fieldForParameter.getter.call(data)!!
+                serde.calculateDataSection(fieldData)
+            },
+            assetName = "MapFile"
         )
     }
 
+    @OptIn(ExperimentalTime::class)
     override fun serialize(outputStream: OutputStream, data: MapFile) {
-        TODO("Not yet implemented")
+
+        // TODO: check if we need to preserve the order in which we write the data
+        parameterToField.entries.forEachIndexed { index, (parameter, fieldForParameter) ->
+            @Suppress("UNCHECKED_CAST")
+            val serde = serdes[index] as Serde<Any>
+            val fieldData = fieldForParameter.getter.call(data)!!
+
+            measureTime {
+                MapFileWriter.writeAsset(outputStream, serializationContext, fieldData)
+                serde.serialize(outputStream, fieldData)
+            }.also { elapsedTime ->
+                if (serializationContext.debugMode) {
+                    println("Deserialization of '${parameter.name}' took $elapsedTime.")
+                }
+            }
+        }
     }
 
     @OptIn(ExperimentalTime::class)
@@ -80,8 +95,6 @@ internal class MapFileSerde(
             }
         }
 
-        return primaryConstructor.call(*values).also {
-            postProcessor.postProcess(it, serializationContext)
-        }
+        return primaryConstructor.call(*values)
     }
 }

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/MapFileWriter.kt

@@ -1,10 +1,12 @@
 package de.darkatra.bfme2.map.serialization
 
 import com.google.common.io.CountingOutputStream
+import de.darkatra.bfme2.map.Asset
 import de.darkatra.bfme2.map.MapFile
 import de.darkatra.bfme2.map.MapFileCompression
 import de.darkatra.bfme2.write7BitIntPrefixedString
 import de.darkatra.bfme2.writeUInt
+import de.darkatra.bfme2.writeUShort
 import java.io.BufferedOutputStream
 import java.io.OutputStream
 import java.io.UnsupportedEncodingException
@@ -15,11 +17,28 @@ import java.util.zip.DeflaterOutputStream
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.exists
 import kotlin.io.path.outputStream
+import kotlin.reflect.full.findAnnotation
 import kotlin.time.ExperimentalTime
 import kotlin.time.measureTime
 
 class MapFileWriter {
 
+    companion object {
+
+        internal fun writeAsset(outputStream: OutputStream, serializationContext: SerializationContext, data: Any) {
+
+            val asset = data::class.findAnnotation<Asset>()
+                ?: throw IllegalStateException("'${data::class.qualifiedName}' must be annotated with '${Asset::class.simpleName}'.")
+
+            val assetIndex = serializationContext.getAssetIndex(asset.name)
+            val assetVersion = asset.version
+
+            outputStream.writeUInt(assetIndex)
+            outputStream.writeUShort(assetVersion)
+            outputStream.writeUInt(serializationContext.getAssetDataSection(asset.name).size.toUInt())
+        }
+    }
+
     @Suppress("unused") // public api
     fun write(file: Path, mapFile: MapFile) {
 
@@ -49,14 +68,22 @@ class MapFileWriter {
                 val annotationProcessingContext = AnnotationProcessingContext(false)
                 val serdeFactory = SerdeFactory(annotationProcessingContext, serializationContext)
 
-                val mapFileSerde = serdeFactory.getSerde(MapFile::class)
+                val mapFileSerde: MapFileSerde = serdeFactory.getSerde(MapFile::class) as MapFileSerde
                 annotationProcessingContext.invalidate()
 
-                // FIXME: correctly calculate asset names from data sections
-                val dataSections = mapFileSerde.collectDataSections(mapFile)
+                val assetDataSections = mapFileSerde.calculateDataSection(mapFile).flatten()
+                    .filter { it.isAsset }
+                    .distinctBy { it.assetName }
+                serializationContext.setAssetDataSections(assetDataSections.associateBy { it.assetName!! })
+
+                val assetNames = assetDataSections
+                    .reversed()
+                    .mapIndexed { index, dataSectionHolder -> Pair(index.toUInt() + 1u, dataSectionHolder.assetName!!) }
+                    .toMap()
+                serializationContext.setAssetNames(assetNames)
 
                 measureTime {
-                    writeAssetNames(mapOf(), countingOutputStream)
+                    writeAssetNames(assetNames, countingOutputStream)
                 }.also { elapsedTime ->
                     if (serializationContext.debugMode) {
                         println("Writing asset names took $elapsedTime.")
@@ -65,8 +92,6 @@ class MapFileWriter {
 
                 mapFileSerde.serialize(bufferedOutputStream, mapFile)
                 bufferedOutputStream.flush()
-
-                serializationContext.pop()
             }
         }
     }

map/src/main/kotlin/de/darkatra/bfme2/map/serialization/MapSerde.kt

@@ -48,20 +48,20 @@ internal class MapSerde<K, V>(
         VALUE_FIRST
     }
 
-    override fun collectDataSections(data: Map<K, V>): DataSection {
+    override fun calculateDataSection(data: Map<K, V>): DataSection {
         return DataSectionHolder(
             containingData = buildList {
                 add(DataSectionLeaf.INT)
                 data.entries.map { (key, value) ->
                     when (order) {
                         DeserializationOrder.KEY_FIRST -> {
-                            add(keySerde.collectDataSections(key))
-                            add(valueSerde.collectDataSections(value))
+                            add(keySerde.calculateDataSection(key))
+                            add(valueSerde.calculateDataSection(value))
                         }
 
                         DeserializationOrder.VALUE_FIRST -> {
-                            add(valueSerde.collectDataSections(value))
-                            add(keySerde.collectDataSections(key))
+                            add(valueSerde.calculateDataSection(value))
+                            add(keySerde.calculateDataSection(key))
                         }
                     }
                 }