diff --git a/Readme.md b/Readme.md index f72ab362..a345c5eb 100644 --- a/Readme.md +++ b/Readme.md @@ -1,5 +1,5 @@ # netchdf -_last updated: 7/9/2025_ +_last updated: 7/13/2025_ This is a rewrite in Kotlin of parts of the devcdm and netcdf-java libraries. diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index 8008dfbf..4f6a0aa9 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -5,9 +5,14 @@ dependencies { api(project(":core")) + implementation(libs.lzf) + implementation(libs.lz4) implementation(libs.kotlinx.cli) implementation(libs.oshai.logging) implementation(libs.logback.classic) + + testImplementation(kotlin("test")) + testImplementation(libs.junit.jupiter.params) } kotlin { diff --git a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/BitShuffleFilter.kt b/cli/src/main/kotlin/com/sunya/netchdf/hdf5/BitShuffleFilter.kt new file mode 100644 index 00000000..c8f480c6 --- /dev/null +++ b/cli/src/main/kotlin/com/sunya/netchdf/hdf5/BitShuffleFilter.kt @@ -0,0 +1,159 @@ +package com.sunya.netchdf.hdf5 + +import net.jpountz.lz4.LZ4Factory + +// seems to handle LZ4_COMPRESSION combined with bit shuffle +class BitShuffleFilter : H5filterIF { + override fun id() = 32008 + override fun name() = "bitshuffle" + val lz4Decompressor = LZ4Factory.fastestJavaInstance().safeDecompressor() + + override fun apply(encodedData: ByteArray, clientValues: IntArray): ByteArray { + val blockSize = if (clientValues[3] == 0) getDefaultBlockSize(clientValues[2]) else clientValues[3] + val blockSizeBytes = blockSize * clientValues[2] + + when (clientValues[4]) { + NO_COMPRESSION -> return noCompression(encodedData, clientValues, blockSizeBytes) + LZ4_COMPRESSION -> return lz4Compression(encodedData, clientValues) + ZSTD_COMPRESSION -> throw RuntimeException("Bitshuffle zstd not implemented") + else -> throw RuntimeException("Unknown compression type: " + clientValues[4]) + } + } + + private fun noCompression(encodedData: ByteArray, clientValues: IntArray, blockSizeBytes: Int): ByteArray { + val nblocks = encodedData.size / blockSizeBytes + val unshuffled = ByteArray(encodedData.size) + for (i in 0..< nblocks) { + val blockData = ByteArray(blockSizeBytes) + System.arraycopy(encodedData, i * blockSizeBytes, blockData, 0, blockSizeBytes) + val unshuffledBlock = ByteArray(blockSizeBytes) + unshuffle(blockData, clientValues[2], unshuffledBlock) + System.arraycopy(unshuffledBlock, 0, unshuffled, i * blockSizeBytes, blockSizeBytes) + } + if (nblocks * blockSizeBytes < encodedData.size) { + val finalBlockSize = encodedData.size - nblocks * blockSizeBytes + val blockData = ByteArray(finalBlockSize) + System.arraycopy(encodedData, nblocks * blockSizeBytes, blockData, 0, finalBlockSize) + val unshuffledBlock = ByteArray(finalBlockSize) + unshuffle(blockData, clientValues[2], unshuffledBlock) + System.arraycopy(unshuffledBlock, 0, unshuffled, nblocks * blockSizeBytes, finalBlockSize) + } + return unshuffled + } + + private fun lz4Compression(encodedData: ByteArray, clientValues: IntArray): ByteArray { + val totalDecompressedSize = Math.toIntExact(makeLongFromBEBytes(encodedData, 0, 8)) + val decompressed = ByteArray(totalDecompressedSize) + + val decompressedBlockSize: Int = makeIntFromBEBytes(encodedData, 8, 4) + val nblocks = if (decompressedBlockSize > totalDecompressedSize) 1 else totalDecompressedSize / decompressedBlockSize + val decompressedBuffer = ByteArray(decompressedBlockSize) + + var srcOffset = 12 + var dstOffset = 0 + + repeat (nblocks) { + val compressedBlockLength = makeIntFromBEBytes(encodedData, srcOffset, 4) + srcOffset += 4 + + val decompressedBytes = lz4Decompressor.decompress(encodedData, srcOffset, compressedBlockLength, decompressedBuffer, 0) + unshuffle(decompressedBuffer, decompressedBytes, decompressed, dstOffset, clientValues[2]) + + srcOffset += compressedBlockLength + dstOffset += decompressedBlockSize + } + + if (dstOffset < totalDecompressedSize) { // copy remaining into destination + encodedData.copyInto(decompressed, destinationOffset = dstOffset, startIndex = srcOffset) + } + + return decompressed + } + + protected fun unshuffle(shuffledBuffer: ByteArray, elementSize: Int, unshuffledBuffer: ByteArray) { + unshuffle(shuffledBuffer, shuffledBuffer.size, unshuffledBuffer, 0, elementSize) + } + + protected fun unshuffle( + shuffledBuffer: ByteArray, + shuffledLength: Int, + unshuffledBuffer: ByteArray, + unshuffledOffset: Int, + elementSize: Int + ) { + val elements = shuffledLength / elementSize + val elementSizeBits = elementSize * 8 + val unshuffledOffsetBits = unshuffledOffset * 8 + + if (elements < 8) { + // https://github.com/xerial/snappy-java/issues/296#issuecomment-964469607 + System.arraycopy(shuffledBuffer, 0, unshuffledBuffer, 0, shuffledLength) + return + } + + val elementsToShuffle = elements - elements % 8 + val elementsToCopy = elements - elementsToShuffle + + var pos = 0 + for (i in 0..= bytes.size * 8)) { "bit index out of range. index=" + bit } + val byteIndex = bit / 8 + val bitInByte = bit % 8 + + // could pregenerate = 1, 2, 4, .. 128 + val bitset = 1 shl bitInByte + val bitclear = (1 shl bitInByte).inv() + + if (value) { + bytes[byteIndex] = (bytes[byteIndex].toInt() or bitset).toByte() + } else { + bytes[byteIndex] = (bytes[byteIndex].toInt() and bitclear).toByte() + } +} diff --git a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/Lz4Filter.kt b/cli/src/main/kotlin/com/sunya/netchdf/hdf5/Lz4Filter.kt new file mode 100644 index 00000000..3878936c --- /dev/null +++ b/cli/src/main/kotlin/com/sunya/netchdf/hdf5/Lz4Filter.kt @@ -0,0 +1,44 @@ +package com.sunya.netchdf.hdf5 + +import net.jpountz.lz4.LZ4Factory +import kotlin.math.min + +class Lz4Filter : H5filterIF { + override fun id() = 32004 + override fun name() = "lz4" + + override fun apply(encodedData: ByteArray, clientValues: IntArray): ByteArray { + val lz4Decompressor = LZ4Factory.fastestJavaInstance().fastDecompressor() + + // https://docs.hdfgroup.org/archive/support/services/filters/HDF5_LZ4.pdf + // bytearray cant be larger than 2^32 + // TODO Big Endian ? + val totalDecompressedSizeLong = makeLongFromBEBytes(encodedData, 0, 8) + val totalDecompressedSize = Math.toIntExact(totalDecompressedSizeLong) + val decompressed = ByteArray(totalDecompressedSize) + + val decompressedBlockSize: Int = makeIntFromBEBytes(encodedData, 8, 4) + val nblocks = (totalDecompressedSize + decompressedBlockSize - 1) / decompressedBlockSize + + var srcOffset = 12 + var dstOffset = 0 + + repeat (nblocks) { + val compressedBlockSize = makeIntFromBEBytes(encodedData, srcOffset, 4) + srcOffset += 4 + val destBlockSize = min(decompressed.size - dstOffset, decompressedBlockSize) + + if (compressedBlockSize == destBlockSize) { + encodedData.copyInto(decompressed, destinationOffset = dstOffset, startIndex = srcOffset, endIndex = srcOffset + destBlockSize) + } else { + // public abstract int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen); + lz4Decompressor.decompress(encodedData, srcOffset, decompressed, dstOffset, destBlockSize) + } + srcOffset += compressedBlockSize + dstOffset += decompressedBlockSize + } + + return decompressed + } + +} \ No newline at end of file diff --git a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/LzfFilter.kt b/cli/src/main/kotlin/com/sunya/netchdf/hdf5/LzfFilter.kt new file mode 100644 index 00000000..278090fe --- /dev/null +++ b/cli/src/main/kotlin/com/sunya/netchdf/hdf5/LzfFilter.kt @@ -0,0 +1,28 @@ +package com.sunya.netchdf.hdf5 + +import com.ning.compress.lzf.LZFException +import com.ning.compress.lzf.util.ChunkDecoderFactory + +class LzfFilter() : H5filterIF { + override fun id() = 32000 + override fun name() = "lzf" + + override fun apply(encodedData: ByteArray, clientValues: IntArray): ByteArray { + val compressedLength = encodedData.size + val uncompressedLength = clientValues[2] + + if (compressedLength == uncompressedLength) { + return encodedData + } + + val output = ByteArray(uncompressedLength) + + try { + ChunkDecoderFactory.safeInstance().decodeChunk(encodedData, 0, output, 0, uncompressedLength) + } catch (e: LZFException) { + throw RuntimeException("Inflating failed", e) + } + return output + } + +} \ No newline at end of file diff --git a/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt b/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt new file mode 100644 index 00000000..aa874582 --- /dev/null +++ b/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt @@ -0,0 +1,82 @@ +package com.sunya.netchdf.hdf5 + +import com.sunya.cdm.api.computeSize +import kotlin.test.Test +import com.sunya.netchdf.openNetchdfFile + +class TestFilters { + init { + FilterRegistrar.registerFilter(Lz4Filter()) + FilterRegistrar.registerFilter(LzfFilter()) + FilterRegistrar.registerFilter(BitShuffleFilter()) + } + + // hdf5 test_compressed_chunked_datasets_earliest.hdf5 { + // + // group: float { + // variables: + // float float32(7, 5) ; + // float float32lzf(7, 5) ; + // double float64(7, 5) ; + // double float64lzf(7, 5) ; + // } + // + // group: int { + // variables: + // short int16(7, 5) ; + // short int16lzf(7, 5) ; + // int int32(7, 5) ; + // int int32lzf(7, 5) ; + // byte int8(7, 5) ; + // byte int8lzf(7, 5) ; + // } + //} + // read float32 + // read float32lzf + // read float64 + // read float64lzf + //failed to read /float/float64lzf, java.lang.RuntimeException: Unimplemented filter type= lzf name = lzf + @Test + fun testLzfFilter() { // 37 + val filename = "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/test_compressed_chunked_datasets_earliest.hdf5" + println(filename) + readNetchdfData(filename, true, true) + } + + // HDF5 "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/lz4_datasets.hdf5" { + //GROUP "/" { + // DATASET "float32_bs0" { + // DATATYPE H5T_IEEE_F32LE + // DATASPACE SIMPLE { ( 20 ) / ( 20 ) } + // } + // .. + @Test + fun testLz4Filter() { // 37 + val filename = "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/lz4_datasets.hdf5" + println(filename) + readNetchdfData(filename, true, true) + } + + @Test + fun testBitShuffleFilter() { // 37 + val filename = "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/bitshuffle_datasets.hdf5" + println(filename) + readNetchdfData(filename, true, true) + } + + fun readNetchdfData(filename: String, showCdl : Boolean = false, showData : Boolean = false) { + openNetchdfFile(filename).use { myfile -> + if (myfile == null) { + println("*** not a netchdf file = $filename") + return + } + println("--- ${myfile.type()} $filename ") + myfile.rootGroup().allVariables().forEach { myvar -> + val mydata = myfile.readArrayData(myvar) + if (showCdl) println(" ${myvar.datatype} ${myvar.fullname()}${myvar.shape.contentToString()} = " + + "${mydata.shape.contentToString()} ${mydata.shape.computeSize()} elems" ) + if (showData) println(mydata) + } + } + } +} \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt new file mode 100644 index 00000000..3d31a167 --- /dev/null +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt @@ -0,0 +1,144 @@ +package com.sunya.netchdf.hdf5 + +import com.sunya.cdm.iosp.decode + +// should work even when filterType unknowm as long as its registered +enum class FilterType(val id: Int) { + none(0), deflate(1), shuffle(2), fletcher32(3), szip(4), nbit(5), scaleoffset(6), + bzip2(307), + lzf(32000), + lz4(32004), + bitshuffle(32008), + zstandard(32015), + unknown(Int.MAX_VALUE); + + companion object { + fun fromId(id: Int): FilterType { + for (type in FilterType.entries) { + if (type.id == id) { + return type + } + } + return unknown + } + + fun nameFromId(id: Int): String { + for (type in entries) { + if (type.id == id) { + return type.name + } + } + return "UnknownFilter$id" + } + } +} + +/** Apply filters, if any. */ +internal class FilterPipeline( + val varname : String, + val mfp: FilterPipelineMessage?, + val isBE: Boolean +) { + + init { + if (mfp != null) { + mfp.filters.forEach { filter -> + if (filter.filterType == FilterType.lz4) { + println("GOT LZ4!") + } + } + } + } + + fun apply(encodedData: ByteArray, filterMask: Int): ByteArray { + if (mfp == null) return encodedData + var data = encodedData + + // apply filters backwards + for (i in mfp.filters.indices.reversed()) { + val filter = mfp.filters[i] + if (isBitSet(filterMask, i)) { + continue + } + + val wtf = findFilter(filter) + if (wtf == null) throw RuntimeException("Unimplemented filter type= ${filter.filterType} name = ${filter.name}") + data = wtf.apply(data, filter.clientValues) + } + return data + } +} + +interface H5filterIF { + fun id() : Int + fun name() : String + fun apply(encodedData: ByteArray, clientValues: IntArray): ByteArray +} + +class DeflateFilter : H5filterIF { + override fun id() = 1 + override fun name() = "deflate" + override fun apply(encodedData: ByteArray, clientValues: IntArray): ByteArray { + return decode(encodedData) + } +} + +class ShuffleFilter() : H5filterIF { + override fun id() = 2 + override fun name() = "shuffle" + override fun apply(encodedData: ByteArray, clientValues: IntArray): ByteArray { + val n = clientValues[0] + if (n <= 1) return encodedData + val m = encodedData.size / n + val count = IntArray(n) + for (k in 0 until n) count[k] = k * m + val result = ByteArray(encodedData.size) + + var pos = 0 + for (i in 0 until m) { + for (j in 0 until n) { + result[i * n + j] = encodedData[i + count[j]] + pos++ + } + } + + // jhdf + if (pos < encodedData.size) { + // In the overrun section no shuffle is done, just a straight copy + // ByteArray.copyInto(destination: ByteArray, destinationOffset: Int = 0, startIndex: Int = 0, endIndex: Int = size) + encodedData.copyInto(result, destinationOffset = pos, startIndex = pos) + } + return result + } +} + +class FletcherFilter : H5filterIF { + override fun id() = 3 + override fun name() = "fletcher" + override fun apply(encodedData: ByteArray, clientValues: IntArray): ByteArray { + // just strip off the 4-byte fletcher32 checksum at the end + // val result = ByteArray(org.size - 4) + // System.arraycopy(org, 0, result, 0, result.size) + // if (debug) println(" checkfletcher32 bytes in= " + org.size + " bytes out= " + result.size) + return encodedData.copyOf(encodedData.size - 4) + } +} + +internal fun findFilter(filterMessage: FilterMessage) : H5filterIF? { + return when (filterMessage.filterType.id) { + 1 -> DeflateFilter() + 2 -> ShuffleFilter() + 3 -> FletcherFilter() + else -> { + FilterRegistrar.resisteredFilters.find { it.id() == filterMessage.filterId } + } + } +} + +object FilterRegistrar { + val resisteredFilters = mutableListOf() + + fun registerFilter(f : H5filterIF) { + resisteredFilters.add(f) + } +} \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt index 66b3515b..024bdab3 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt @@ -143,8 +143,8 @@ internal class FractalHeap(private val h5: H5builder, forWho: String, address: L // The length of the object in the heap, determined by taking the minimum value of // Maximum Direct Block Size and Maximum Size of Managed Objects in the Fractal Heap Header. // Again, the minimum number of bytes needed to encode that value is used for the size of this field. - offset = makeIntFromBytes(heapId, 1, n) - size = makeIntFromBytes(heapId, 1 + n, m) + offset = makeIntFromLEBytes(heapId, 1, n) + size = makeIntFromLEBytes(heapId, 1 + n, m) } 1 -> { // how fun to guess the subtype @@ -152,7 +152,7 @@ internal class FractalHeap(private val h5: H5builder, forWho: String, address: L val hasFilters = (ioFilterLen > 0) subtype = if (hasBtree) if (hasFilters) 2 else 1 else if (hasFilters) 4 else 3 when (subtype) { - 1, 2 -> offset = makeIntFromBytes(heapId, 1, (heapId.size - 1)) + 1, 2 -> offset = makeIntFromLEBytes(heapId, 1, (heapId.size - 1)) } } 2 -> { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt index d51ba518..704f57ac 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt @@ -12,8 +12,8 @@ import com.fleeksoft.charset.Charsets import com.sunya.cdm.array.makeString import com.sunya.cdm.util.InternalLibraryApi -private const val debugStart = false -private const val debugSuperblock = false +private const val debugStart = true +private const val debugSuperblock = true internal const val debugTypedefs = false internal const val debugFlow = false diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt index e68a20f5..5ece17c5 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt @@ -20,7 +20,7 @@ internal class H5chunkIterator(val h5 : H5builder, val v2: Variable, val w val elemSize : Int val datatype : Datatype<*> val tiledData : H5TiledData1 - val filters : H5filters + val filters : FilterPipeline val state : OpenFileState private val wantSpace : IndexSpace @@ -33,7 +33,7 @@ internal class H5chunkIterator(val h5 : H5builder, val v2: Variable, val w val btreeNew = BTree1(h5, vinfo.dataPos, 1, vinfo.storageDims.size) tiledData = H5TiledData1(btreeNew, v2.shape, vinfo.storageDims) - filters = H5filters(v2.name, vinfo.mfp, h5type.isBE) + filters = FilterPipeline(v2.name, vinfo.mfp, h5type.isBE) if (debugChunking) println(" H5chunkIterator tiles=${tiledData.tiling}") state = OpenFileState(0L, h5type.isBE) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt index 3c045d58..35805738 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt @@ -29,7 +29,7 @@ internal class H5chunkReader(val h5 : H5builder) { // just reading into memory the entire index for now // val index = BTree2j(h5, v2.name, vinfo.dataPos, vinfo.storageDims) - val filters = H5filters(v2.name, vinfo.mfp, vinfo.h5type.isBE) + val filters = FilterPipeline(v2.name, vinfo.mfp, vinfo.h5type.isBE) val state = OpenFileState(0L, vinfo.h5type.isBE) // just run through all the chunks, we wont read any that we dont want @@ -79,7 +79,7 @@ internal class H5chunkReader(val h5 : H5builder) { throw RuntimeException("Unsupprted mdl ${vinfo.mdl}") val tiledData = H5TiledData1(btree, v2.shape, vinfo.storageDims) - val filters = H5filters(v2.name, vinfo.mfp, vinfo.h5type.isBE) + val filters = FilterPipeline(v2.name, vinfo.mfp, vinfo.h5type.isBE) if (debugChunking) println(" readChunkedData tiles=${tiledData.tiling}") var transferChunks = 0 diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5filters.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5filters.kt deleted file mode 100644 index b4390f97..00000000 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5filters.kt +++ /dev/null @@ -1,131 +0,0 @@ -package com.sunya.netchdf.hdf5 - -import com.sunya.cdm.iosp.decode - - -internal data class Filter(val filterType: FilterType, val name: String, val clientValues: IntArray) - -internal data class FilterPipelineMessage(val filters: List) : MessageHeader(MessageType.FilterPipeline) { - override fun show() : String { - return filters.joinToString { "${it.filterType} ${it.name}, " } - } -} - -/** Apply filters, if any. */ -internal class H5filters( - val varname : String, - val mfp: FilterPipelineMessage?, - val isBE: Boolean -) { - - fun apply(rawdata: ByteArray, filterMask : Int): ByteArray { - if (mfp == null) return rawdata - - var data = rawdata - // apply filters backwards - for (i in mfp.filters.indices.reversed()) { - val filter = mfp.filters[i] - if (isBitSet(filterMask, i)) { - continue - } - data = when (filter.filterType) { - FilterType.deflate -> { - // val oldway = inflate(data) - decode(data) - } - - FilterType.shuffle -> shuffle(data, filter.clientValues[0]) - - FilterType.fletcher32 -> checkfletcher32(data) - - /* FilterType.zstandard -> { - val result = zstandard(data, expectedLengthBytes) - result.order(byteOrder) - return result // LOOK end of filters ?? - } */ - else -> throw RuntimeException("Unknown filter type= ${filter.filterType} name = ${filter.name}") - } - } - return data - } - - /* - * decompress using Zstandard - * - * @param compressed compressed data - * @return uncompressed data - * - private fun zstandard(compressed: ByteArray, expectedLengthBytes: Int): ByteBuffer { - val input = ByteBuffer.wrap(compressed) - val output = ByteBuffer.wrap(ByteArray(expectedLengthBytes)) - val decompressor = ZstdDecompressor() - decompressor.decompress(input, output) - output.flip() - if (debug || debugFilter) { - val compress = compressed.size.toFloat() / output.limit() - System.out.printf( - " zstandard bytes in= %d out= %d compress = %f.2%n", compressed.size, output.limit(), - compress - ) - } - return output.slice() - } - - */ - -/* - private fun inflate(compressed: ByteArray): ByteArray { - // run it through the Inflator - val input = ByteArrayInputStream(compressed) - val inflater = Inflater() - val inflatestream = InflaterInputStream(input, inflater, inflateBufferSize) - val len = min(8 * compressed.size, MAX_ARRAY_LEN) - val out = ByteArrayOutputStream(len) // Fixes KXL-349288 - IOcopyB(inflatestream, out, inflateBufferSize) - val uncomp = out.toByteArray() - if (debug || debugFilter) println(" inflate bytes in= " + compressed.size + " bytes out= " + uncomp.size) - return uncomp - } - - */ - - // just strip off the 4-byte fletcher32 checksum at the end - private fun checkfletcher32(org: ByteArray): ByteArray { - // val result = ByteArray(org.size - 4) - // System.arraycopy(org, 0, result, 0, result.size) - // if (debug) println(" checkfletcher32 bytes in= " + org.size + " bytes out= " + result.size) - return org.copyOf(org.size - 4) - } - - private fun shuffle(data: ByteArray, n: Int): ByteArray { - if (debug) println(" shuffle bytes in= " + data.size + " n= " + n) - // require(data.size % n == 0) - if (n <= 1) return data - val m = data.size / n - val count = IntArray(n) - for (k in 0 until n) count[k] = k * m - val result = ByteArray(data.size) - - var pos = 0 - for (i in 0 until m) { - for (j in 0 until n) { - result[i * n + j] = data[i + count[j]] - pos++ - } - } - - // jhdf - if (pos < data.size) { - // In the overrun section no shuffle is done, just a straight copy - data.copyInto(result, destinationOffset = pos, startIndex = pos) - } - return result - } - - - companion object { - var debugFilter = false - private const val MAX_ARRAY_LEN = Int.MAX_VALUE - 8 - private const val debug = false - } -} \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt index 37b04ada..be12ce25 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt @@ -62,9 +62,9 @@ internal fun H5builder.readGroupNew( val btree2j = BTree2j(this, parent.name, btreeAddress) for (record in btree2j.records) { val heapId: ByteArray = when (btree2j.btreeType) { - 8 -> (record as BTree2j.Record8).heapId - 9 -> (record as BTree2j.Record9).heapId - else -> continue + 5 -> (record as BTree2j.Record5).heapId + 6 -> (record as BTree2j.Record6).heapId + else -> throw RuntimeException("btree2 type ${btree2j.btreeType} mot supported") } // the heapId points to a Link message in the Fractal Heap diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt index 2de51315..521a4e1d 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt @@ -530,7 +530,7 @@ internal fun H5builder.readFilterPipelineMessage(state: OpenFileState): FilterPi if (version == 1) { state.pos += 6 } - val filters = mutableListOf() + val filters = mutableListOf() for (i in 0 until nfilters) { val filterId = raf.readShort(state).toInt() val filterType = FilterType.fromId(filterId) @@ -547,40 +547,24 @@ internal fun H5builder.readFilterPipelineMessage(state: OpenFileState): FilterPi if (version == 1 && nValues and 1 != 0) { // check if odd state.pos += 4 } - filters.add(Filter(filterType, name, clientValues)) + val f = FilterMessage(filterId, filterType, name, clientValues) + filters.add(f) } return FilterPipelineMessage(filters) } -// H5Xpublic.h -internal enum class FilterType(val id: Int) { - none(0), deflate(1), shuffle(2), fletcher32(3), szip(4), nbit(5), scaleoffset(6), - bzip2(307), - zstandard(32015), - unknown(Int.MAX_VALUE); +internal class FilterMessage(val filterId: Int, val filterType: FilterType, val name: String, val clientValues: IntArray) - companion object { - fun fromId(id: Int): FilterType { - for (type in FilterType.entries) { - if (type.id == id) { - return type - } - } - return unknown - } - - fun nameFromId(id: Int): String { - for (type in entries) { - if (type.id == id) { - return type.name - } - } - return "UnknownFilter$id" - } +internal data class FilterPipelineMessage(val filters: List) : MessageHeader(MessageType.FilterPipeline) { + override fun show() : String { + return filters.joinToString { "${it.filterType} ${it.name}, " } } } +// H5Xpublic.h + + ///////////////////////////////////////////// 12/0xC "Attribute" : define an Attribute // The Attribute message is used to store objects in the HDF5 file which are used as attributes, or “metadata” about // the current object. An attribute is a small dataset; it has a name, a datatype, a dataspace, and raw data. diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Util.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Util.kt index 25b46070..1d032992 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Util.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Util.kt @@ -4,7 +4,7 @@ fun isBitSet(num: Int, bitno: Int): Boolean { return ((num ushr bitno) and 1) != 0 } -fun makeIntFromBytes(bb: ByteArray, start: Int, n: Int): Int { +fun makeIntFromLEBytes(bb: ByteArray, start: Int, n: Int): Int { var result = 0 for (i in start + n - 1 downTo start) { result = result shl 8 @@ -12,4 +12,34 @@ fun makeIntFromBytes(bb: ByteArray, start: Int, n: Int): Int { result += if ((b < 0)) b + 256 else b } return result +} + +fun makeLongFromLEBytes(bb: ByteArray, start: Int, n: Int): Long { + var result = 0L + for (i in start + n - 1 downTo start) { + result = result shl 8 + val b = bb[i].toInt() + result += if ((b < 0)) b + 256 else b + } + return result +} + +fun makeIntFromBEBytes(bb: ByteArray, start: Int, n: Int): Int { + var result = 0 + for (i in start until start + n) { + result = result shl 8 + val b = bb[i].toInt() + result += if ((b < 0)) b + 256 else b + } + return result +} + +fun makeLongFromBEBytes(bb: ByteArray, start: Int, n: Int): Long { + var result = 0L + for (i in start until start + n) { + result = result shl 8 + val b = bb[i].toInt() + result += if ((b < 0)) b + 256 else b + } + return result } \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a3a7ef37..b1454175 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,6 +7,8 @@ kotlinx-cli = "0.3.6" kotlinx-coroutines = "1.10.2" fleeksoft-version = "0.0.4" +lzf-version = "1.1.2" +lz4-version = "1.8.0" okio-version = "3.12.0" oshai-version = "7.0.0" @@ -26,6 +28,9 @@ slf4j = "1.7.36" okio = { module = "com.squareup.okio:okio", version.ref = "okio-version" } fleeksoft = { module = "com.fleeksoft.charset:charset", version.ref = "fleeksoft-version" } +lzf = { module = "com.ning:compress-lzf", version.ref = "lzf-version" } +lz4 = { module = "org.lz4:lz4-java", version.ref = "lz4-version" } + # Kotlinx libraries kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" } kotlinx-cli = { module = "org.jetbrains.kotlinx:kotlinx-cli", version.ref = "kotlinx-cli" } diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/JhdfReadTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/JhdfReadTest.kt index 091a4dd0..d5bb625c 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/JhdfReadTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/JhdfReadTest.kt @@ -72,7 +72,6 @@ class JhdfReadTest { readNetchdfData(filename, null, null, true, false) } - @Test fun testFractalHeap() { // 28 val filename = "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/test_attribute_latest.hdf5" @@ -80,6 +79,14 @@ class JhdfReadTest { readNetchdfData(filename, null, null, true, false) } + // Unknown filter type= lzf name = lzf + @Test + fun testLzfFilter() { // 37 + val filename = "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/test_compressed_chunked_datasets_earliest.hdf5" + println(filename) + readNetchdfData(filename, null, null, true, false) + } + // private typedefs @Test fun testVlen() { // 63 @@ -88,34 +95,6 @@ class JhdfReadTest { readNetchdfData(filename, null, null, true, false) } - // HDF5 "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/test_compound_scalar_attribute.hdf5" { - //GROUP "/" { - // GROUP "GROUP" { - // ATTRIBUTE "VERSION" { - // DATATYPE H5T_COMPOUND { - // H5T_STD_I32LE "myMajor"; - // H5T_STD_I32LE "myMinor"; - // H5T_STD_I32LE "myPatch"; - // } - // DATASPACE SCALAR - // } - // } - //} - //} - // hdf5 test_compound_scalar_attribute.hdf5 { - // - // group: GROUP { - // types: - // compound anon { - // int myMajor ; - // int myMinor ; - // int myPatch ; - // }; // anon - // - // // group attributes: - // :VERSION = {myMajor = 1, myMinor = 0, myPatch = 0} ; - // } - //} @Test fun testCompoundAttribute() { val filename = "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/test_compound_scalar_attribute.hdf5" @@ -123,125 +102,6 @@ class JhdfReadTest { readNetchdfData(filename, null, null, true, false) } - // "committed" aka "named" datatype - // HDF5 "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/issue255_example.hdf5" { - //GROUP "/" { - // GROUP "__DATA_TYPES__" { - // DATATYPE "Enum_Boolean" H5T_ENUM { - // H5T_STD_I8LE; - // "FALSE" 0; - // "TRUE" 1; - // }; - // DATATYPE "String_VariableLength" H5T_STRING { - // STRSIZE H5T_VARIABLE; - // STRPAD H5T_STR_NULLTERM; - // CSET H5T_CSET_ASCII; - // CTYPE H5T_C_S1; - // }; - // } - // GROUP "groupA" { - // DATASET "date" { - // DATATYPE H5T_STD_I64LE - // DATASPACE SCALAR - // ATTRIBUTE "__TYPE_VARIANT__" { - // DATATYPE H5T_ENUM { - // H5T_STD_I8LE; - // "TIMESTAMP_MILLISECONDS_SINCE_START_OF_THE_EPOCH" 0; - // "TIME_DURATION_MICROSECONDS" 1; - // "TIME_DURATION_MILLISECONDS" 2; - // "TIME_DURATION_SECONDS" 3; - // "TIME_DURATION_MINUTES" 4; - // "TIME_DURATION_HOURS" 5; - // "TIME_DURATION_DAYS" 6; - // "ENUM" 7; - // "NONE" 8; - // "BITFIELD" 9; - // } - // DATASPACE SCALAR - // } - // } - // GROUP "groupC" { - // } - // DATASET "string" { - // DATATYPE H5T_STRING { - // STRSIZE 24; - // STRPAD H5T_STR_NULLPAD; - // CSET H5T_CSET_ASCII; - // CTYPE H5T_C_S1; - // } - // DATASPACE SCALAR - // } - // } - // GROUP "groupB" { - // ATTRIBUTE "__TYPE_VARIANT__timestamp__" { - // DATATYPE H5T_ENUM { - // H5T_STD_I8LE; - // "TIMESTAMP_MILLISECONDS_SINCE_START_OF_THE_EPOCH" 0; - // "TIME_DURATION_MICROSECONDS" 1; - // "TIME_DURATION_MILLISECONDS" 2; - // "TIME_DURATION_SECONDS" 3; - // "TIME_DURATION_MINUTES" 4; - // "TIME_DURATION_HOURS" 5; - // "TIME_DURATION_DAYS" 6; - // "ENUM" 7; - // "NONE" 8; - // "BITFIELD" 9; - // } - // DATASPACE SCALAR - // } - // ATTRIBUTE "important" { - // DATATYPE "/__DATA_TYPES__/Enum_Boolean" - // DATASPACE SCALAR - // } - // ATTRIBUTE "timestamp" { - // DATATYPE H5T_STD_I64LE - // DATASPACE SCALAR - // } - // DATASET "dmat" { - // DATATYPE H5T_IEEE_F64LE - // DATASPACE SIMPLE { ( 3, 3 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } - // } - // SOFTLINK "groupC" { - // LINKTARGET "/groupA/groupC" - // } - // DATASET "inarr" { - // DATATYPE H5T_STD_I32LE - // DATASPACE SIMPLE { ( 3 ) / ( H5S_UNLIMITED ) } - // } - // } - //} - //} - // hdf5 issue255_example.hdf5 { - // types: - // ubyte enum anon {0 = TIMESTAMP_MILLISECONDS_SINCE_START_OF_THE_EPOCH, 1 = TIME_DURATION_MICROSECONDS, 2 = TIME_DURATION_MILLISECONDS, 3 = TIME_DURATION_SECONDS, 4 = TIME_DURATION_MINUTES, 5 = TIME_DURATION_HOURS, 6 = TIME_DURATION_DAYS, 7 = ENUM, 8 = NONE, 9 = BITFIELD}; - // - // group: __DATA_TYPES__ { - // types: - // ubyte enum Enum_Boolean {0 = FALSE, 1 = TRUE}; - // ubyte(*) String_VariableLength ; - // } - // - // group: groupA { - // variables: - // int64 date ; - // :__TYPE_VARIANT__ = TIMESTAMP_MILLISECONDS_SINCE_START_OF_THE_EPOCH ; - // string string ; - // - // group: groupC { - // } - // } - // - // group: groupB { - // variables: - // double dmat(3, 3) ; - // int inarr(3) ; - // - // // group attributes: - // :__TYPE_VARIANT__timestamp__ = TIMESTAMP_MILLISECONDS_SINCE_START_OF_THE_EPOCH ; - // :important = FALSE ; - // :timestamp = 1550033296762 ; - // } - //} @Test fun testCommittedDatatype() { val filename = "/home/stormy/dev/github/netcdf/jhdf/jhdf/src/test/resources/hdf5/issue255_example.hdf5"