added cleanup option for mpgs, added recovery option, added debug option

README.md

@@ -14,6 +14,15 @@ Then extract your MPGs by running
 ```
 java -jar hddfsdump.jar --input ./hddfs.bin --output ./mpg-out/
 ```
+
+| Params  | Default | Description | Environment Var |
+| ------------- | ------------- | ------------- | ------------- |
+| --input   | | Specifies the input file  | HDDFS_DUMP_IN
+| --output  | | Specifies the output directory for the final MPGs  | HDDFS_DUMP_OUT
+| --no-mpg-cleanup | false | Cleaning up MPGs means skipping non sequential blocks. Some VCRs simply override deleted MPGs with new ones, resulting in an overlap at the end of the mpg. If you encounter issues with the cleanup you can use this option to disable it.  | HDDFS_DUMP_NO_CLEANUP
+| --recover  | false | Dump even deleted data. You might want to use this in combination with --no-mpg-cleanup  | HDDFS_DUMP_RECOVER
+| --debug  | false | Dont write to disk, debug output on console  | HDDFS_DUMP_DEBUG
+
 
 ## Dependencies:
 * [Clikt](https://github.com/ajalt/clikt) Copyright 2018-2020 AJ Alt, Apache License Version 2.0  

src/Main.kt

@@ -1,15 +1,18 @@
 import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.options.required
 import com.github.ajalt.clikt.parameters.types.file
+import fs.HDDFs
 import fs.HDDFsImage
 import logger.ConsoleLogger
+import logger.EmptyLogger
 import utils.blocksToMpgSize
 import utils.format
 import java.io.File
 import kotlin.system.exitProcess
 
-const val VERSION = 0.1
+const val VERSION = 0.2
 
 @Suppress("SpellCheckingInspection")
 class Main : CliktCommand(printHelpOnEmptyArgs = true) {
@@ -26,23 +29,46 @@ class Main : CliktCommand(printHelpOnEmptyArgs = true) {
         canBeDir = true
     ).required()
 
+    private val debug: Boolean by option(
+        "--debug", envvar = "HDDFS_DUMP_DEBUG",
+        help = "Dont write to disk, debug output on console"
+    ).flag(default = false)
+
+    private val noMpgCleanup: Boolean by option(
+        "--no-mpg-cleanup", envvar = "HDDFS_DUMP_NO_CLEANUP",
+        help = "Cleaning up MPGs means skipping non sequential blocks. " +
+                "Some VCRs simply override deleted MPGs with new ones, resulting in an overlap at the end of the mpg." +
+                "If you encounter issues with the cleanup you can use this option to disable it."
+    ).flag(default = false)
+
+    private val recover: Boolean by option(
+        "--recover", envvar = "HDDFS_DUMP_RECOVER",
+        help = "Dump even deleted data. You might want to use this in combination with --no-mpg-cleanup"
+    ).flag(default = false)
+
     override fun run() {
         intro()
 
         val pathToFile = input.toString()
         val outPath = output.toString()
 
-        val hddfs = loadHDDFSImage(pathToFile)
+        val hddfs = loadHDDFSImage(pathToFile, debug)
         checkValidHDDFs(hddfs)
 
         val mpgPosition = hddfs.findMpgSectionPosition().also {
             println("MPG sector found at: ${it.first}, length: ${it.second} ")
         }
 
-        val mpgBlocks = getMPGBlockCount(hddfs, mpgPosition)
+        val mpgBlocks = getMPGBlockCount(hddfs, mpgPosition, recover)
 
-        println("Extracting to \"$outPath\"...")
-        hddfs.extractAllMpgs(outPath, mpgPosition, ConsoleLogger(mpgBlocks))
+        println("Extracting to \"$outPath\"${ if(noMpgCleanup) " (no cleanup)" else "" }...")
+        hddfs.extractAllMpgs(
+            outPath,
+            mpgPosition,
+            if (debug) EmptyLogger(mpgBlocks) else ConsoleLogger(mpgBlocks),
+            noMpgCleanup,
+            recover
+        )
     }
 
     private fun intro() {
@@ -55,18 +81,24 @@ class Main : CliktCommand(printHelpOnEmptyArgs = true) {
         println("- Progressbar, Copyright 2015-2020 Tongfei Chen, The MIT License (https://github.com/ctongfei/progressbar)\n")
     }
 
-    private fun getMPGBlockCount(hddfs: HDDFsImage, mpgPosition: Pair<UInt, UInt>): Long {
+    private fun getMPGBlockCount(hddfs: HDDFsImage, mpgPosition: Pair<UInt, UInt>, recover: Boolean): Long {
         print("Analysing MPG sector (this might take a while)... ")
-        val mpgBlocks = hddfs.getMpgBlockCount(mpgPosition)
+
+        val mpgBlocks = if(recover){
+            mpgPosition.second.toLong() / HDDFs.MPG_ALLOC_BLOCK_SECTORS
+        } else {
+            hddfs.getMpgBlockCount(mpgPosition)
+        }
         val mpgSize = mpgBlocks.blocksToMpgSize
+
         println("OK")
         println("Found ${mpgSize.toULong().format()} bytes of MPG data (Blocks: $mpgBlocks)\n")
         return mpgBlocks
     }
 
-    private fun loadHDDFSImage(pathToFile: String): HDDFsImage {
+    private fun loadHDDFSImage(pathToFile: String, debug: Boolean): HDDFsImage {
         print("Reading \"$pathToFile\"... ")
-        return HDDFsImage(pathToFile).also {
+        return HDDFsImage(pathToFile, debug).also {
             println("OK")
         }
     }

src/fs/HDDFs.kt

@@ -4,10 +4,12 @@ import utils.sectorAsPosition
 
 object HDDFs {
 
-    /** MPG Allocation Block Size (MPGs are stored in ~8MB Blocks) **/
-    const val MPG_ALLOC_BLOCK_SECTORS = 16384L
+    /** MPG Allocation Block Size (MPGs are stored in Blocks) **/
+    const val MPG_ALLOC_BLOCK_SECTORS = 512L
 
-    /** Same as MPG_ALLOC_BLOCK_SECTORS only in bytes instead of sectors. ~8MB **/
+    const val MPG_ALLOC_BLOCK_SECTORS_ANALYSE = 8192L
+
+    /** Same as MPG_ALLOC_BLOCK_SECTORS only in bytes instead of sectors.**/
     val MPG_ALLOC_BLOCK_SIZE = MPG_ALLOC_BLOCK_SECTORS.sectorAsPosition
 
     /** Contains the HDDFs signature that is found in SYSCTR  **/
@@ -19,7 +21,10 @@ object HDDFs {
     /** HDD Sector size in bytes **/
     const val HDD_SECTOR_SIZE = 512
 
-    /** This signature starts after the MPG Signature (00 00 01 BA) and signals a new file start **/
+    /** MPG Signature **/
+    val MPG_SIGNATURE = byteArrayOf(0x00.toByte(), 0x00.toByte(), 0x01.toByte(), 0xBA.toByte())
+
+    /** This signature starts after the MPG_SIGNATURE (00 00 01 BA) and signals a new file start **/
     val MPG_NEW_SIGNATURE = byteArrayOf(0x44, 0x00, 0x04, 0x00, 0x04, 0x01)
 
     /** Empty array, used to check whether we reached end of MPG data **/

src/fs/HDDFsImage.kt

@@ -2,10 +2,10 @@ package fs
 
 import logger.ProgressLogger
 import utils.sectorAsPosition
-import utils.toUInt32
+import utils.toUInt32LittleEndian
 
 @Suppress("MemberVisibilityCanBePrivate")
-class HDDFsImage(pathToFile: String) : FileSeeker(pathToFile) {
+class HDDFsImage(pathToFile: String, private val debug: Boolean) : FileSeeker(pathToFile) {
 
     fun getSignatureSector() = readBytes(HDDFs.SYSCTR.first, HDDFs.SYSCTR.second.toInt())
 
@@ -35,24 +35,24 @@ class HDDFsImage(pathToFile: String) : FileSeeker(pathToFile) {
         check(rawRelativePosition != -1) { "Could not locate MPG position" }
 
         val absolutePosition = rawRelativePosition - HDDFs.MPG_LOCATION_OFFSET_TO_NAME + HDDFs.FB000.first - 1
-        val mpgStart = readBytes(absolutePosition, 4).toUInt32()
-        val mpgLength = readBytes(absolutePosition + 4, 4).toUInt32()
+        val mpgStart = readBytes(absolutePosition, 4).toUInt32LittleEndian()
+        val mpgLength = readBytes(absolutePosition + 4, 4).toUInt32LittleEndian()
 
         return Pair(mpgStart, mpgLength)
     }
 
     /**
-     * Skipping through MPG_ALLOC_BLOCKS and see if we read empty (zero) bytes
+     * Skipping through MPG_ALLOC_BLOCK_SECTORS_ANALYSE and see if we read empty (zero) bytes
      * @param position sector position + length of MPG sector (can be retrieved by calling [findMpgSectionPosition])
-     * @return last block number with non empty data
+     * @return last block number with non empty data (approximation for speed)
      */
     fun getMpgBlockCount(position: Pair<UInt, UInt>): Long {
-        for (i in 0 until position.second.toInt() / HDDFs.MPG_ALLOC_BLOCK_SECTORS) {
-            val absolutePosition = (position.first.toLong() + (i * HDDFs.MPG_ALLOC_BLOCK_SECTORS)).sectorAsPosition
+        for (i in 0 until position.second.toInt() / HDDFs.MPG_ALLOC_BLOCK_SECTORS_ANALYSE) {
+            val absolutePosition = (position.first.toLong() + (i * HDDFs.MPG_ALLOC_BLOCK_SECTORS_ANALYSE)).sectorAsPosition
             val chunk = readBytes(absolutePosition, HDDFs.EMPTY.size)
-            if (HDDFs.EMPTY.findIn(chunk, 0)) return (i - 1L)
+            if (HDDFs.EMPTY.findIn(chunk, 0)) return (i - 1L) * (HDDFs.MPG_ALLOC_BLOCK_SECTORS_ANALYSE / HDDFs.MPG_ALLOC_BLOCK_SECTORS)
         }
-        return position.second.toLong()
+        return position.second.toLong() / HDDFs.MPG_ALLOC_BLOCK_SECTORS
     }
 
     /**
@@ -61,8 +61,8 @@ class HDDFsImage(pathToFile: String) : FileSeeker(pathToFile) {
      * @param position sector position + length of MPG sector (can be retrieved with [findMpgSectionPosition])
      * @param logger used only for progress bar, can be retrieved by calling [getMpgBlockCount]
      */
-    fun extractAllMpgs(outPath: String, position: Pair<UInt, UInt>, logger: ProgressLogger) {
-        MpgScratcher(this, position, logger).extractTo(outPath)
+    fun extractAllMpgs(outPath: String, position: Pair<UInt, UInt>, logger: ProgressLogger, allowSequenceJumps: Boolean, recover: Boolean) {
+        MpgScratcher(this, position, logger, allowSequenceJumps, recover, debug).extractTo(outPath)
     }
 
     companion object {

src/fs/MpgScratcher.kt

@@ -3,14 +3,18 @@ package fs
 import fs.HDDFsImage.Companion.findIn
 import logger.ProgressLogger
 import utils.sectorAsPosition
+import utils.toUInt32BigEndian
 import java.io.File
 import java.io.FileOutputStream
 
 @Suppress("ConstantConditionIf", "SpellCheckingInspection")
 class MpgScratcher(
     private val source: HDDFsImage,
     private val position: Pair<UInt, UInt>,
-    private val logger: ProgressLogger
+    private val logger: ProgressLogger,
+    private val allowSequenceJumps: Boolean, // noMpgCleanup
+    private val recover: Boolean,
+    private val debug: Boolean = false
 ) {
 
     fun extractTo(outPath: String) {
@@ -21,11 +25,14 @@ class MpgScratcher(
     }
 
     private fun exportAll(outPath: String, offset: Int, fileOffset: Int) {
+        if(debug) println("$fileOffset.mpg starts at index $offset")
+
         val nextOffset = write("$outPath/$fileOffset.mpg", offset)
-        if (nextOffset == COMPLETE) {
+        if (nextOffset == COMPLETE || nextOffset == position.second.toInt() / HDDFs.MPG_ALLOC_BLOCK_SECTORS.toInt()) {
             logger.complete()
             return
         }
+
         exportAll(outPath, nextOffset, fileOffset + 1)
     }
 
@@ -37,34 +44,152 @@ class MpgScratcher(
             fos = FileOutputStream(file)
 
             var i = offset
+            var lastSequence = HDDFs.MPG_NEW_SIGNATURE.toUInt32BigEndian()
+            var lastGap = 7u * HDDFs.MPG_ALLOC_BLOCK_SECTORS.toUInt()
+
             while (i in offset until position.second.toInt() / HDDFs.MPG_ALLOC_BLOCK_SECTORS) {
                 logger.progress(i.toLong(), file.name)
 
-                val absolutePosition = (position.first.toLong() + (i * HDDFs.MPG_ALLOC_BLOCK_SECTORS)).sectorAsPosition
-                val chunk = source.readBytes(absolutePosition, HDDFs.MPG_ALLOC_BLOCK_SIZE.toInt())
+                // Current Block
+                val chunk = readChunk(i)
+                val isMpgBlock = hasMpgHeader(chunk)
+                val sequenceNumber = getSequenceNumber(chunk)
+
+                // Next Block
+                val nextChunkHeader = readChunk(i + 1, 12)
+                val nextIsMpgBlock = hasMpgHeader(nextChunkHeader)
+                val nextSequence = getSequenceNumber(nextChunkHeader)
 
-                // Reached end of mpg
-                if (HDDFs.EMPTY.findIn(chunk, 0)) {
+                // Abort criterium 1: Reached end of mpg, we are done
+                val isEmpty = HDDFs.EMPTY.findIn(chunk, 0)
+                if (!recover && isEmpty) {
                     i = COMPLETE
                     break
                 }
 
-                // Reached end of file, write next one
+                // Abort criterium 2: Reached end of title, finish this write up and continue with next one
                 if (i != offset && HDDFs.MPG_NEW_SIGNATURE.findIn(chunk, 4)) {
                     break
                 }
 
-                fos.write(chunk)
+                // Protect against ghost data (VCR overrides previously deleted mpg)
+                // Ignore those sudden sequence jumps if allowSequenceJumps is not set
+                val margin = lastGap + (lastGap * 4u).toUInt()
+                val changeToNow = sequenceNumber - lastSequence
+                if(!isMpgBlock || isSequenceSequential(changeToNow, margin) || allowSequenceJumps){
+                    if(debug){
+                        println("$isMpgBlock (" + sequenceNumber + ") " + debugGetHeader(chunk) +" || margin: $margin || change: $changeToNow")
+                    } else {
+                        // Cleanup last block
+                        if(nextIsMpgBlock && !isSequenceSequential(nextSequence - sequenceNumber, margin * 3u) && !allowSequenceJumps) {
+                            fos.write(cleanupLastBlock(chunk, sequenceNumber))
+                        } else {
+                            if(!isEmpty) fos.write(chunk)
+                        }
+                    }
+
+                    if(isMpgBlock){
+                        lastGap = sequenceNumber - lastSequence
+                        lastSequence = sequenceNumber
+                    }
+                }
+
                 i++
             }
 
-            fos.flush()
+            if(!debug) fos.flush()
             return i
         } finally {
             fos?.close()
         }
     }
 
+    /**
+     * So the VCR overrides old deleted MPGs with new onces which is why we need to pay attention
+     * to the sequence number. Since we read in bigger chunks for performance reasons we manually
+     * clean up the last block if we find a non sequential subblock within the last block
+     * @param chunk read last block
+     * @param sequenceNumber of last block
+     * @return either the whole chunk if no sequential inconsistency has been found
+     * or a subset of chunk of all sequential subblocks if it is inconsistent
+     */
+    private fun cleanupLastBlock(chunk: ByteArray, sequenceNumber: UInt): ByteArray{
+        var lastGap = HDDFs.HDD_SECTOR_SIZE.toUInt()
+        var lastSequence = sequenceNumber
+
+        for(i in 1 until chunk.size / HDDFs.HDD_SECTOR_SIZE){
+            val sequence = getSequenceNumber(chunk, (i * HDDFs.HDD_SECTOR_SIZE) + 4)
+            val isMpgBlock = hasMpgHeader(chunk, i * HDDFs.HDD_SECTOR_SIZE)
+            val changeToNow = sequence - lastSequence
+
+            val margin = lastGap + (lastGap * 4u).toUInt()
+            if(isMpgBlock && ! isSequenceSequential(changeToNow, margin)){
+                val newChunk = ByteArray(i * HDDFs.HDD_SECTOR_SIZE)
+                chunk.copyInto(newChunk, 0, 0, newChunk.size)
+                if(debug) println("Last block was cleaned up")
+                return newChunk
+            }
+
+            if(isMpgBlock){
+                lastGap = sequence - lastSequence
+                lastSequence = sequence
+            }
+        }
+        return chunk
+    }
+
+    /**
+     * Has the following block a valid MPG signature
+     * @param chunk read block
+     * @param offset optional offset
+     * @return true if chunk has a valid MPG signature at given offset
+     */
+    private fun hasMpgHeader(chunk: ByteArray, offset: Int = 0): Boolean {
+        return HDDFs.MPG_SIGNATURE.findIn(chunk, offset)
+    }
+
+    /**
+     * @param change difference between last block and this block
+     * @param margin allowed margin to be considered a sequential block
+     * @returns true if difference is within the margin (sequential)
+     * or false if not
+     */
+    private fun isSequenceSequential(change: UInt, margin: UInt): Boolean {
+        return change <= margin && change > 0u
+    }
+
+    /**
+     * Read a block at a given offset with given length
+     * @param offset in sectors
+     * @param length in size
+     * @returns block of data at offset with length
+     */
+    private fun readChunk(offset: Int, length: Int = HDDFs.MPG_ALLOC_BLOCK_SIZE.toInt()): ByteArray {
+        val absolutePosition = (position.first.toLong() + (offset * HDDFs.MPG_ALLOC_BLOCK_SECTORS)).sectorAsPosition
+        return source.readBytes(absolutePosition, length)
+    }
+
+    /**
+     * Get the sequence number of a MPG block
+     * @param chunk mpg block
+     * @param offset on where the sequence number is located in the mpg block. Default offset of 4
+     * @return an UInt interpretation of the 4 most significant bytes of the mpg sequence
+     */
+    private fun getSequenceNumber(chunk: ByteArray, offset: Int = 4): UInt {
+        val sequenceNumber = ByteArray(4)
+        chunk.copyInto(sequenceNumber, 0, offset, endIndex = sequenceNumber.size + offset)
+        return sequenceNumber.toUInt32BigEndian()
+    }
+
+    private fun debugGetHeader(chunk: ByteArray): String {
+        var erg = ""
+        val until = if(chunk.size > 13) 13 else chunk.size
+        for(i in 0 until until){
+            erg += String.format("%02x", chunk[i]) + " "
+        }
+        return erg
+    }
+
     companion object {
         const val COMPLETE = -1
     }