From 489f306e8e61dda1bb57733697131bdcfa3aa518 Mon Sep 17 00:00:00 2001 From: Roman Chernyatchik Date: Sat, 31 Mar 2018 00:46:42 +0300 Subject: [PATCH] - `RAFBufferFactory`, `RAFBuffer`, `RomBufferFactory` moved to `org.jetbrains.bio` package - `RAFBufferFactory` supports buffer size option for underlying random access file - By default `RAFBufferFactory` uses default Random Access File buffers size (8092 bytes) instead of 125 kb. --- CHANGES | 6 ++++++ .../kotlin/org/jetbrains/bio/MMBRomBuffer.kt | 1 - .../org/jetbrains/bio/{big => }/RAFBuffer.kt | 21 ++++++++++++------- .../org/jetbrains/bio/RandomAccessFile.kt | 18 +++++++++------- .../bio/{big => }/RomBufferFactory.kt | 4 +--- .../kotlin/org/jetbrains/bio/tdf/TdfFile.kt | 4 ++-- 6 files changed, 32 insertions(+), 22 deletions(-) rename src/main/kotlin/org/jetbrains/bio/{big => }/RAFBuffer.kt (84%) rename src/main/kotlin/org/jetbrains/bio/{big => }/RomBufferFactory.kt (63%) diff --git a/CHANGES b/CHANGES index 23bd0a1..1550dbc 100644 --- a/CHANGES +++ b/CHANGES @@ -11,6 +11,12 @@ Version 0.7.1 Changed - Use maxHeapSize = "1024m" option for tests. Seems by default on windows heap size in about 256m and it isn't enough for tests +- `RAFBufferFactory`, `RAFBuffer`, `RomBufferFactory` moved to + `org.jetbrains.bio` package +- `RAFBufferFactory` supports buffer size option for underlying + random access file +- By default `RAFBufferFactory` uses default Random Access File + buffers size (8092 bytes) instead of 125 kb. Version 0.7.0 ------------- diff --git a/src/main/kotlin/org/jetbrains/bio/MMBRomBuffer.kt b/src/main/kotlin/org/jetbrains/bio/MMBRomBuffer.kt index 1c4a6f0..1db2e48 100644 --- a/src/main/kotlin/org/jetbrains/bio/MMBRomBuffer.kt +++ b/src/main/kotlin/org/jetbrains/bio/MMBRomBuffer.kt @@ -2,7 +2,6 @@ package org.jetbrains.bio import com.google.common.primitives.* import com.indeed.util.mmap.MMapBuffer -import org.jetbrains.bio.big.RomBufferFactory import java.nio.ByteOrder import java.nio.channels.FileChannel import java.nio.file.Path diff --git a/src/main/kotlin/org/jetbrains/bio/big/RAFBuffer.kt b/src/main/kotlin/org/jetbrains/bio/RAFBuffer.kt similarity index 84% rename from src/main/kotlin/org/jetbrains/bio/big/RAFBuffer.kt rename to src/main/kotlin/org/jetbrains/bio/RAFBuffer.kt index cd1cbb2..bca0846 100644 --- a/src/main/kotlin/org/jetbrains/bio/big/RAFBuffer.kt +++ b/src/main/kotlin/org/jetbrains/bio/RAFBuffer.kt @@ -1,12 +1,16 @@ -package org.jetbrains.bio.big +package org.jetbrains.bio -import org.jetbrains.bio.RandomAccessFile -import org.jetbrains.bio.RomBuffer import java.nio.ByteOrder import java.nio.file.Path -class RAFBufferFactory(private val path: Path, private val byteOrder: ByteOrder): RomBufferFactory { - override fun create(): RomBuffer = RAFBuffer(path, byteOrder) +/** + * @param path File path + * @param byteOrder Byte order + * @param bufferSize Random access file buffer size in bytes, use -1 for default value + */ +class RAFBufferFactory(private val path: Path, private val byteOrder: ByteOrder, + val bufferSize: Int = -1): RomBufferFactory { + override fun create(): RomBuffer = RAFBuffer(path, byteOrder, bufferSize = bufferSize) override fun close() { // Do nothing @@ -16,9 +20,10 @@ class RAFBufferFactory(private val path: Path, private val byteOrder: ByteOrder) class RAFBuffer(private val path: Path, override val order: ByteOrder, position: Long = 0L, - limit: Long = -1) : RomBuffer() { + limit: Long = -1, + val bufferSize: Int = -1) : RomBuffer() { - private val randomAccessFile = RandomAccessFile(path.toAbsolutePath().toString(), 128000).apply { + private val randomAccessFile = RandomAccessFile(path.toAbsolutePath().toString(), bufferSize).apply { order(order) seek(position) } @@ -37,7 +42,7 @@ class RAFBuffer(private val path: Path, get() = randomAccessFile.filePointer set(position) { randomAccessFile.seek(position) } - override fun duplicate() = RAFBuffer(path, order, position, limit) + override fun duplicate() = RAFBuffer(path, order, position, limit, bufferSize) override fun readInts(size: Int): IntArray { val dst = IntArray(size) diff --git a/src/main/kotlin/org/jetbrains/bio/RandomAccessFile.kt b/src/main/kotlin/org/jetbrains/bio/RandomAccessFile.kt index 783dc64..bbe6187 100644 --- a/src/main/kotlin/org/jetbrains/bio/RandomAccessFile.kt +++ b/src/main/kotlin/org/jetbrains/bio/RandomAccessFile.kt @@ -97,8 +97,9 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe /** * The buffer used for reading the data. + * Will be set to null on file close */ - protected lateinit var buffer: ByteArray + protected var buffer: ByteArray? = null /** * The offset in bytes of the start of the buffer, from the start of the file. @@ -134,7 +135,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe * Internal buffer size in bytes. If writing, call flush() first. */ var bufferSize: Int - get() = buffer.size + get() = buffer!!.size set(bufferSize) { bufferStart = 0 dataEnd = 0 @@ -202,6 +203,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe // Close the underlying file object. randomAccessFile!!.close() randomAccessFile = null // help the gc + buffer = null // help the gc } /** @@ -230,7 +232,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe bufferStart = pos filePointer = pos - dataSize = read_(pos, buffer, 0, buffer.size) + dataSize = read_(pos, buffer!!, 0, buffer!!.size) if (dataSize <= 0) { dataSize = 0 @@ -285,7 +287,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe filePointer < dataEnd -> { val pos = (filePointer - bufferStart).toInt() filePointer++ - (buffer[pos].toInt() and 0xff) + (buffer!![pos].toInt() and 0xff) } @@ -333,7 +335,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe len else bytesAvailable - System.arraycopy(buffer, (filePointer - bufferStart).toInt(), b, off, copyLength) + System.arraycopy(buffer!!, (filePointer - bufferStart).toInt(), b, off, copyLength) filePointer += copyLength.toLong() // If there is more to copy... @@ -342,7 +344,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe // If the amount remaining is more than a buffer's length, read it // directly from the file. - if (extraCopy > buffer.size) { + if (extraCopy > buffer!!.size) { extraCopy = read_(filePointer, b, off + copyLength, len - copyLength) // ...or read a new buffer full, and copy as much as possible... @@ -353,7 +355,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe dataSize else extraCopy - System.arraycopy(buffer, 0, b, off + copyLength, extraCopy) + System.arraycopy(buffer!!, 0, b, off + copyLength, extraCopy) } else { extraCopy = -1 } @@ -1043,7 +1045,7 @@ open class RandomAccessFile(val location: String, bufferSize: Int = defaultBuffe if (debugAccess) { if (logRead) - LOG.debug(" **read_ " + location + " = " + len + " bytes at " + pos + "; block = " + pos / buffer.size) + LOG.debug(" **read_ " + location + " = " + len + " bytes at " + pos + "; block = " + pos / buffer!!.size) seeksCounter.incrementAndGet() bytesReadCounter.addAndGet(len.toLong()) } diff --git a/src/main/kotlin/org/jetbrains/bio/big/RomBufferFactory.kt b/src/main/kotlin/org/jetbrains/bio/RomBufferFactory.kt similarity index 63% rename from src/main/kotlin/org/jetbrains/bio/big/RomBufferFactory.kt rename to src/main/kotlin/org/jetbrains/bio/RomBufferFactory.kt index 9f62ef5..e5e4187 100644 --- a/src/main/kotlin/org/jetbrains/bio/big/RomBufferFactory.kt +++ b/src/main/kotlin/org/jetbrains/bio/RomBufferFactory.kt @@ -1,6 +1,4 @@ -package org.jetbrains.bio.big - -import org.jetbrains.bio.RomBuffer +package org.jetbrains.bio /** * @author Roman.Chernyatchik diff --git a/src/main/kotlin/org/jetbrains/bio/tdf/TdfFile.kt b/src/main/kotlin/org/jetbrains/bio/tdf/TdfFile.kt index d538087..b2a2fb3 100644 --- a/src/main/kotlin/org/jetbrains/bio/tdf/TdfFile.kt +++ b/src/main/kotlin/org/jetbrains/bio/tdf/TdfFile.kt @@ -3,8 +3,8 @@ package org.jetbrains.bio.tdf import com.google.common.primitives.Ints import org.jetbrains.bio.CompressionType import org.jetbrains.bio.RomBuffer -import org.jetbrains.bio.big.RAFBufferFactory -import org.jetbrains.bio.big.RomBufferFactory +import org.jetbrains.bio.RAFBufferFactory +import org.jetbrains.bio.RomBufferFactory import org.jetbrains.bio.divCeiling import org.jetbrains.bio.mapUnboxed import java.io.Closeable