diff --git a/Readme.md b/Readme.md index b156de51..debfb418 100644 --- a/Readme.md +++ b/Readme.md @@ -233,7 +233,7 @@ Currently using * Netcdf-c library version: 4.10.0-development of May 23 2025 * HDF-4 library version: HDF Version 4.2 Release 17-1, March 8, 2023 -In order to run, you must install the C libraries on your computer and ad them to the LD_LIBRARY_PATH. +In order to run these tests, you must install the C libraries on your computer and add them to the LD_LIBRARY_PATH. ### Data Model notes diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt index 96edb572..78772ac8 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt @@ -13,7 +13,8 @@ internal class BTree1data( rootNodeAddress: Long, varShape: LongArray, chunkShape: LongArray, -) { +) : DataChunkSequence { + val tiling = Tiling(varShape, chunkShape) val ndimStorage = chunkShape.size val rootNode: BTreeNode @@ -23,11 +24,11 @@ internal class BTree1data( } // if other layouts like BTree2data had this interface we could use in chunkConcurrent - fun asSequence(): Sequence> = sequence { + override fun asSequence(): Sequence = sequence { repeat( tiling.nelems) { //val startingIndex = tiling.orderToIndex(it.toLong()) //val indexSpace = IndexSpace(startingIndex, tiling.chunk) - yield(Pair(it.toLong(), findDataChunk(it) ?: missingDataChunk(it))) + yield(findDataChunk(it) ?: missingDataChunk(it)) } } @@ -35,7 +36,6 @@ internal class BTree1data( return rootNode.findDataChunk(order) } - // here both internal and leaf are the same structure // Btree nodes Level 1A1 - Version 1 B-trees inner class BTreeNode(val address: Long, val parent: BTreeNode?) { var level: Int = 0 @@ -105,18 +105,17 @@ internal class BTree1data( data class DataChunkKey(val order: Int, val chunkSize: Int, val filterMask : Int) - // childAddress = data chunk (level 1) else a child node inner class DataChunk(val key : DataChunkKey, val childAddress : Long) : DataChunkIF { override fun childAddress() = childAddress override fun offsets() = tiling.orderToIndex(key.order.toLong()) override fun isMissing() = (childAddress <= 0L) // may be 0 or -1 override fun chunkSize() = key.chunkSize override fun filterMask() = key.filterMask + override fun show() = show(tiling) - override fun show(tiling : Tiling) : String = "order=$key, chunkSize=${key.chunkSize}, chunkStart=${offsets().contentToString()}" + + fun show(tiling : Tiling) : String = "order=$key, chunkSize=${key.chunkSize}, chunkStart=${offsets().contentToString()}" + ", tile= ${tiling.tile(offsets() ).contentToString()}" - fun show() = show(tiling) } fun missingDataChunk(order: Int) : DataChunk { @@ -124,13 +123,3 @@ internal class BTree1data( } } -interface DataChunkIF { - fun childAddress(): Long - fun offsets(): LongArray - fun isMissing(): Boolean - fun chunkSize(): Int - fun filterMask(): Int? - - fun show(tiling : Tiling): String -} - diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2.kt new file mode 100644 index 00000000..013976fc --- /dev/null +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2.kt @@ -0,0 +1,240 @@ +package com.sunya.netchdf.hdf5 + +import com.sunya.cdm.iosp.OpenFileState + +import com.sunya.cdm.util.InternalLibraryApi +import com.sunya.cdm.util.log2 +import kotlin.math.ceil +import kotlin.math.pow + +@OptIn(InternalLibraryApi::class) + +/* Btree version 2, for non-data. From jhdf. */ +internal class BTree2(val raf: OpenFileExtended, val owner: String, address: Long) { + + val btreeType: Int + private val nodeSize: Int // size in bytes of btree nodes + private val recordSize: Int // size in bytes of btree records + val treeDepth : Int + val rootNodeAddress: Long + val numberOfRecordsInRoot : Int + val totalNumberOfRecordsInTree: Long + + /** The records in this b-tree */ + val records = mutableListOf() + + init { + val state = OpenFileState(raf.getFileOffset(address), false) + + // header + val magic = raf.readString(state, 4) + check(magic == "BTHD") { "$magic should equal BTHD" } + val version: Byte = raf.readByte(state) + btreeType = raf.readByte(state).toInt() + require(btreeType > 0 || btreeType < 10) + + nodeSize = raf.readInt(state) // This is the size in bytes of all B-tree nodes. + recordSize = raf.readShort(state).toUShort().toInt() // This field is the size in bytes of the B-tree record. + treeDepth = raf.readShort(state).toUShort().toInt() + + val splitPct = raf.readByte(state) + val mergePct = raf.readByte(state) + rootNodeAddress = raf.readOffset(state) + numberOfRecordsInRoot = raf.readShort(state).toUShort().toInt() + totalNumberOfRecordsInTree = raf.readLength(state) // total in entire btree + val checksum: Int = raf.readInt(state) + + readRecords(rootNodeAddress, treeDepth, numberOfRecordsInRoot, totalNumberOfRecordsInTree) + } + + fun readRecords(address: Long, depth: Int, numberOfRecords: Int, totalRecords: Long) { + val state = OpenFileState(raf.getFileOffset(address), false) + + val magic = raf.readString(state, 4) + val leafNode = if (magic == "BTIN") { + false + } else if (magic == "BTLF") { + true + } else { + throw RuntimeException("$magic unknown tag") + } + + val version: Byte = raf.readByte(state) + val nodeType = raf.readByte(state).toInt() // same as the B-tree type in the header + check(nodeType == btreeType) + + repeat(numberOfRecords) { + records.add( readRecord(state, nodeType)) + } + + if (!leafNode) { + repeat(numberOfRecords + 1) { + val childAddress = raf.readOffset(state) // Child Node Pointer + val sizeOfNumberOfRecords = getSizeOfNumberOfRecords(nodeSize, depth, totalRecords.toInt(), recordSize, raf.sizeOffsets()) + val numberOfChildRecords: Int = raf.readVariableSizeUnsigned(state, sizeOfNumberOfRecords).toInt() // readBytesAsUnsignedInt(bb, sizeOfNumberOfRecords) + val sizeNumberOfChildRecords = getSizeOfTotalNumberOfChildRecords(nodeSize, depth, recordSize) + val totalNumberOfChildRecords = if (depth > 1) { + raf.readVariableSizeUnsigned(state, sizeNumberOfChildRecords) + } else { + -1 + } + readRecords(childAddress, depth - 1, numberOfChildRecords, totalNumberOfChildRecords) + } + } + } + + fun readRecord(state: OpenFileState, type: Int): Any { + return when (type) { + 1 -> Record1(state) + 2 -> Record2(state) + 3 -> Record3(state) + 4 -> Record4(state) + 5 -> Record5(state) + 6 -> Record6(state) + 7 -> Record70(state) // TODO wrong + 8 -> Record8(state) + 9 -> Record9(state) + else -> throw IllegalStateException() + } + } + + // Type 1 Record Layout - Indirectly Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects + internal inner class Record1(state: OpenFileState) { + val hugeObjectAddress = raf.readOffset(state) + val hugeObjectLength = raf.readLength(state) + val hugeObjectID = raf.readLength(state) + } + + // Type 2 Record Layout - Indirectly Accessed, Filtered, ‘Huge’ Fractal Heap Objects + internal inner class Record2(state: OpenFileState) { + val hugeObjectAddress = raf.readOffset(state) + val hugeObjectLength = raf.readLength(state) + val filterMask = raf.readInt(state) + val hugeObjectSize = raf.readLength(state) + val hugeObjectID = raf.readLength(state) + } + + // Type 3 Record Layout - Directly Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects + internal inner class Record3(state: OpenFileState) { + val hugeObjectAddress = raf.readOffset(state) + val hugeObjectLength = raf.readLength(state) + } + + // Type 4 Record Layout - Directly Accessed, Filtered, ‘Huge’ Fractal Heap Objects + internal inner class Record4(state: OpenFileState) { + val hugeObjectAddress = raf.readOffset(state) + val hugeObjectLength = raf.readLength(state) + val filterMask = raf.readInt(state) + val hugeObjectSize = raf.readLength(state) + } + + // Type 5 Record Layout - Link Name for Indexed Group + inner class Record5(state: OpenFileState) { + val nameHash = raf.readInt(state) + val heapId: ByteArray = raf.readByteArray(state, 7) + } + + // Type 6 Record Layout - Creation Order for Indexed Group + inner class Record6(state: OpenFileState) { + val creationOrder = raf.readLong(state) + val heapId: ByteArray = raf.readByteArray(state, 7) + } + + // Type 7 Record Layout - Shared Object Header Messages (Sub-type 0 - Message in Heap) + internal inner class Record70(state: OpenFileState) { + val location = raf.readByte(state) + val hash = raf.readInt(state) + val refCount = raf.readInt(state) + val id: ByteArray = raf.readByteArray(state, 8) + } + + // Type 7 Record Layout - Shared Object Header Messages (Sub-type 1 - Message in Object Header) + internal inner class Record71(state: OpenFileState) { + val location = raf.readByte(state) + val hash = raf.readInt(state) + val skip = raf.readByte(state) + val messtype = raf.readByte(state) + val index = raf.readShort(state) + val address = raf.readOffset(state) + } + + // Type 8 Record Layout - Attribute Name for Indexed Attributes + inner class Record8(state: OpenFileState) { + val heapId: ByteArray = raf.readByteArray(state, 8) + val flags = raf.readByte(state) + val creationOrder = raf.readInt(state) + val nameHash = raf.readInt(state) + } + + // Type 9 Record Layout - Creation Order for Indexed Attributes + inner class Record9(state: OpenFileState) { + val heapId: ByteArray = raf.readByteArray(state, 8) + val flags = raf.readByte(state) + val creationOrder = raf.readInt(state) + } + + companion object { + internal fun findRecord1byId(records: List, hugeObjectID: Int): Record1? { + for (record in records) { + if (record is Record1 && record.hugeObjectID == hugeObjectID.toLong()) return record + } + return null + } + } +} + + +// heroic jhdf +fun getSizeOfNumberOfRecords( + nodeSize: Int, + depth: Int, + totalRecords: Int, + recordSize: Int, + sizeOfOffsets: Int +): Int { + val NODE_OVERHEAD_BYTES = 10 + var size: Int = nodeSize - NODE_OVERHEAD_BYTES + + // If the child is not a leaf + if (depth > 1) { + // Need to subtract the pointers as well + val pointerTripletBytes = bytesNeededToHoldNumber(totalRecords) * 2 + sizeOfOffsets + size -= pointerTripletBytes + + return bytesNeededToHoldNumber(size / recordSize) + } else { + // Its a leaf + return bytesNeededToHoldNumber(size / recordSize) + } +} + +// jhdf +internal fun bytesNeededToHoldNumber(number: Int): Int { + return (Integer.numberOfTrailingZeros(Integer.highestOneBit(number)) + 8) / 8 +} + +/* private fun getSizeOfTotalNumberOfChildRecords(nodeSize: Int, depth: Int, recordSize: Int): Int { + require (nodeSize % recordSize == 0) + val recordsInLeafNode = (nodeSize / recordSize).toDouble() + val totalRecords = recordsInLeafNode.pow(depth) + val totalBits = log2(totalRecords) + val totalBitsInt = totalBits.toInt() + return (totalBitsInt + 8) / 8 +} */ + +// no BigInteger, max depth 6 +internal fun getSizeOfTotalNumberOfChildRecords(nodeSize: Int, depth: Int, recordSize: Int): Int { + require(depth < 7 ) { "no BigInteger, max depth 6 "} + val recordsInLeafNode = (nodeSize/ recordSize).toDouble() + val totalRecords = recordsInLeafNode.pow(depth) + val totalRecordsL = ceil(totalRecords).toLong() + val alt = log2(totalRecordsL) + 1 + val alt1 = (alt + 8) / 8 + return alt1 +} + +// jhdf +//private fun getSizeOfTotalNumberOfChildRecordsOrg(nodeSize: Int, depth: Int, recordSize: Int): Int { +// val recordsInLeafNode = nodeSize / recordSize +// return (BigInteger.valueOf(recordsInLeafNode.toLong()).pow(depth).bitLength() + 8) / 8 +//} diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt index bd016e31..dde01ce5 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt @@ -2,336 +2,212 @@ package com.sunya.netchdf.hdf5 import com.sunya.cdm.api.computeSize import com.sunya.cdm.api.toIntArray -import com.sunya.cdm.iosp.OpenFileIF +import com.sunya.cdm.api.toLongArray import com.sunya.cdm.iosp.OpenFileState import com.sunya.cdm.layout.Tiling import com.sunya.cdm.util.InternalLibraryApi -import com.sunya.cdm.util.log2 -import kotlin.math.ceil -import kotlin.math.pow @OptIn(InternalLibraryApi::class) /* Btree version 2, for data. From jhdf. */ -internal class BTree2data(private val h5: H5builder, owner: String, address: Long, storageDims: LongArray? = null) { // BTree2 +internal class BTree2data( + val raf: OpenFileExtended, + val owner: String, + address: Long, + varShape: LongArray, + storageDims: LongArray, +) : DataChunkSequence { // BTree2 + + val chunkSize = storageDims.computeSize() + val chunkShape = LongArray(storageDims.size - 1) { storageDims[it] } + val tiling = Tiling(varShape, chunkShape) + val btreeType: Int private val nodeSize: Int // size in bytes of btree nodes private val recordSize: Int // size in bytes of btree records - private val owner: String - private val raf: OpenFileIF - val rootNodeAddress: Long val treeDepth : Int + val rootNodeAddress: Long val numberOfRecordsInRoot : Int - val totalNumberOfRecordsInTree: Long + val totalNumberOfRecordsInTree: Int - val chunkSize: Long - val chunkDims: LongArray - - /** The records in this b-tree */ - val records = mutableListOf() + val rootNode: BTreeNode init { - if (storageDims != null) { - chunkSize = storageDims.computeSize() - chunkDims = LongArray(storageDims.size - 1) { storageDims[it] } - } else { - chunkSize = 0 - chunkDims = longArrayOf() - } - - raf = h5.raf - this.owner = owner - val state = OpenFileState(h5.getFileOffset(address), false) + val state = OpenFileState(raf.getFileOffset(address), false) // header val magic = raf.readString(state, 4) check(magic == "BTHD") { "$magic should equal BTHD" } val version: Byte = raf.readByte(state) btreeType = raf.readByte(state).toInt() + require(btreeType == 10 || btreeType == 11) + nodeSize = raf.readInt(state) // This is the size in bytes of all B-tree nodes. recordSize = raf.readShort(state).toUShort().toInt() // This field is the size in bytes of the B-tree record. treeDepth = raf.readShort(state).toUShort().toInt() val splitPct = raf.readByte(state) val mergePct = raf.readByte(state) - rootNodeAddress = h5.readOffset(state) + rootNodeAddress = raf.readOffset(state) numberOfRecordsInRoot = raf.readShort(state).toUShort().toInt() - totalNumberOfRecordsInTree = h5.readLength(state) // total in entire btree + totalNumberOfRecordsInTree = raf.readLength(state).toInt() // total in entire btree val checksum: Int = raf.readInt(state) - readRecords(rootNodeAddress, treeDepth, numberOfRecordsInRoot, totalNumberOfRecordsInTree) - } - - fun readRecords(address: Long, depth: Int, numberOfRecords: Int, totalRecords: Long) { - val state = OpenFileState(h5.getFileOffset(address), false) - - val magic = raf.readString(state, 4) - val leafNode = if (magic == "BTIN") { - false - } else if (magic == "BTLF") { - true - } else { - throw RuntimeException("$magic unknown tag") - } - - val version: Byte = raf.readByte(state) - val nodeType = raf.readByte(state).toInt() // same as the B-tree type in the header - check(nodeType == btreeType) - - repeat(numberOfRecords) { - records.add( readRecord(state, nodeType)) - } - - if (!leafNode) { - repeat(numberOfRecords + 1) { - val childAddress = h5.readOffset(state) // Child Node Pointer - val sizeOfNumberOfRecords = getSizeOfNumberOfRecords(nodeSize, depth, totalRecords.toInt(), recordSize, h5.sizeOffsets) - val numberOfChildRecords: Int = h5.readVariableSizeUnsigned(state, sizeOfNumberOfRecords).toInt() // readBytesAsUnsignedInt(bb, sizeOfNumberOfRecords) - val sizeNumberOfChildRecords = getSizeOfTotalNumberOfChildRecords(nodeSize, depth, recordSize) - val totalNumberOfChildRecords = if (depth > 1) { - h5.readVariableSizeUnsigned(state, sizeNumberOfChildRecords) - } else { - -1 - } - readRecords(childAddress, depth - 1, numberOfChildRecords, totalNumberOfChildRecords) - } - } + rootNode = BTreeNode(rootNodeAddress, treeDepth, numberOfRecordsInRoot, totalNumberOfRecordsInTree, null) } - // heroic jhdf - private fun getSizeOfNumberOfRecords( - nodeSize: Int, - depth: Int, - totalRecords: Int, - recordSize: Int, - sizeOfOffsets: Int - ): Int { - val NODE_OVERHEAD_BYTES = 10 - var size: Int = nodeSize - NODE_OVERHEAD_BYTES - - // If the child is not a leaf - if (depth > 1) { - // Need to subtract the pointers as well - val pointerTripletBytes = bytesNeededToHoldNumber(totalRecords) * 2 + sizeOfOffsets - size -= pointerTripletBytes - - return bytesNeededToHoldNumber(size / recordSize) - } else { - // Its a leaf - return bytesNeededToHoldNumber(size / recordSize) + override fun asSequence(): Sequence = sequence { + repeat( tiling.nelems) { + //val startingIndex = tiling.orderToIndex(it.toLong()) + //val indexSpace = IndexSpace(startingIndex, tiling.chunk) + yield(findDataChunk(it) ?: missingDataChunk(it)) } } - // jhdf - private fun bytesNeededToHoldNumber(number: Int): Int { - return (Integer.numberOfTrailingZeros(Integer.highestOneBit(number)) + 8) / 8 - } + fun chunkIterator(): Iterator = asSequence().iterator() - /* private fun getSizeOfTotalNumberOfChildRecords(nodeSize: Int, depth: Int, recordSize: Int): Int { - require (nodeSize % recordSize == 0) - val recordsInLeafNode = (nodeSize / recordSize).toDouble() - val totalRecords = recordsInLeafNode.pow(depth) - val totalBits = log2(totalRecords) - val totalBitsInt = totalBits.toInt() - return (totalBitsInt + 8) / 8 - } */ - - // no BigInteger, max depth 6 - private fun getSizeOfTotalNumberOfChildRecords(nodeSize: Int, depth: Int, recordSize: Int): Int { - require(depth < 7 ) { "no BigInteger, max depth 6 "} - val recordsInLeafNode = (nodeSize/ recordSize).toDouble() - val totalRecords = recordsInLeafNode.pow(depth) - val totalRecordsL = ceil(totalRecords).toLong() - val alt = log2(totalRecordsL) + 1 - val alt1 = (alt + 8) / 8 - return alt1 + internal fun findDataChunk(order: Int): DataChunkIF? { + return rootNode.findDataChunk(order) } - // jhdf - //private fun getSizeOfTotalNumberOfChildRecordsOrg(nodeSize: Int, depth: Int, recordSize: Int): Int { - // val recordsInLeafNode = nodeSize / recordSize - // return (BigInteger.valueOf(recordsInLeafNode.toLong()).pow(depth).bitLength() + 8) / 8 - //} + inner class BTreeNode(val address: Long, depth: Int, numberOfRecords: Int, totalRecords: Int, val parent: BTreeNode?) { + var level: Int = 0 + var nentries: Int = 0 - fun readRecord(state: OpenFileState, type: Int): Any { - return when (type) { - 1 -> Record1(state) - 2 -> Record2(state) - 3 -> Record3(state) - 4 -> Record4(state) - 5 -> Record5(state) - 6 -> Record6(state) - 7 -> Record70(state) // TODO wrong - 8 -> Record8(state) - 9 -> Record9(state) - 10 -> Record10(state, chunkDims.toIntArray(), chunkSize.toInt()) - 11 -> Record11(state, chunkDims.toIntArray() ) - else -> throw IllegalStateException() - } - } + val keyValues = mutableListOf>() // tile order to DataChunk + val children = mutableListOf() - // Type 1 Record Layout - Indirectly Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects - internal inner class Record1(state: OpenFileState) { - val hugeObjectAddress = h5.readOffset(state) - val hugeObjectLength = h5.readLength(state) - val hugeObjectID = h5.readLength(state) - } + var lastOrder : Int = 0 - // Type 2 Record Layout - Indirectly Accessed, Filtered, ‘Huge’ Fractal Heap Objects - internal inner class Record2(state: OpenFileState) { - val hugeObjectAddress = h5.readOffset(state) - val hugeObjectLength = h5.readLength(state) - val filterMask = raf.readInt(state) - val hugeObjectSize = h5.readLength(state) - val hugeObjectID = h5.readLength(state) - } + init { + if (address > 0) { + val state = OpenFileState(raf.getFileOffset(address), false) + + val magic = raf.readString(state, 4) + val leafNode = if (magic == "BTIN") { + false + } else if (magic == "BTLF") { + true + } else { + throw RuntimeException("$magic unknown tag") + } - // Type 3 Record Layout - Directly Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects - internal inner class Record3(state: OpenFileState) { - val hugeObjectAddress = h5.readOffset(state) - val hugeObjectLength = h5.readLength(state) - } + val version: Byte = raf.readByte(state) + val nodeType = raf.readByte(state).toInt() // same as the B-tree type in the header + check(nodeType == btreeType) - // Type 4 Record Layout - Directly Accessed, Filtered, ‘Huge’ Fractal Heap Objects - internal inner class Record4(state: OpenFileState) { - val hugeObjectAddress = h5.readOffset(state) - val hugeObjectLength = h5.readLength(state) - val filterMask = raf.readInt(state) - val hugeObjectSize = h5.readLength(state) - } + // dataChunks + repeat(numberOfRecords) { + val chunkImpl = readRecord(state, nodeType) + val order = tiling.order(chunkImpl.chunkOffset.toLongArray()).toInt() + keyValues.add(Pair(order, chunkImpl)) + lastOrder = order + } - // Type 5 Record Layout - Link Name for Indexed Group - inner class Record5(state: OpenFileState) { - val nameHash = raf.readInt(state) - val heapId: ByteArray = raf.readByteArray(state, 7) - } + // children + if (!leafNode) { + repeat(numberOfRecords + 1) { + val childAddress = raf.readOffset(state) // Child Node Pointer + val sizeOfNumberOfRecords = getSizeOfNumberOfRecords(nodeSize, depth, totalRecords.toInt(), recordSize, raf.sizeOffsets()) + val numberOfChildRecords: Int = raf.readVariableSizeUnsigned(state, sizeOfNumberOfRecords).toInt() // readBytesAsUnsignedInt(bb, sizeOfNumberOfRecords) + val sizeNumberOfChildRecords = getSizeOfTotalNumberOfChildRecords(nodeSize, depth, recordSize) + val totalNumberOfChildRecords = if (depth > 1) { + raf.readVariableSizeUnsigned(state, sizeNumberOfChildRecords).toInt() + } else { + -1 + } + children.add( BTreeNode(childAddress, depth - 1, numberOfChildRecords, totalNumberOfChildRecords, this)) + } + } - // Type 6 Record Layout - Creation Order for Indexed Group - inner class Record6(state: OpenFileState) { - val creationOrder = raf.readLong(state) - val heapId: ByteArray = raf.readByteArray(state, 7) - } + if (children.isNotEmpty()) { + lastOrder = children.last().lastOrder + } + } + } - // Type 7 Record Layout - Shared Object Header Messages (Sub-type 0 - Message in Heap) - internal inner class Record70(state: OpenFileState) { - val location = raf.readByte(state) - val hash = raf.readInt(state) - val refCount = raf.readInt(state) - val id: ByteArray = raf.readByteArray(state, 8) - } + // uses a tree search = O(log n) + fun findDataChunk(wantOrder: Int): DataChunkIF? { + if (children.isNotEmpty()) { // search tree; assumes that chunks are ordered + children.forEach { childNode -> + if (wantOrder <= childNode.lastOrder) + return childNode.findDataChunk(wantOrder) + } + } else { // If it's a leaf node (no children) + val kv = keyValues.find { it.first == wantOrder } + return kv?.second + } + return null + } - // Type 7 Record Layout - Shared Object Header Messages (Sub-type 1 - Message in Object Header) - internal inner class Record71(state: OpenFileState) { - val location = raf.readByte(state) - val hash = raf.readInt(state) - val skip = raf.readByte(state) - val messtype = raf.readByte(state) - val index = raf.readShort(state) - val address = h5.readOffset(state) - } + override fun toString(): String { + return "BTreeNode(address=$address, level=$level, nentries=$nentries, lastOrder=$lastOrder)" + } - // Type 8 Record Layout - Attribute Name for Indexed Attributes - inner class Record8(state: OpenFileState) { - val heapId: ByteArray = raf.readByteArray(state, 8) - val flags = raf.readByte(state) - val creationOrder = raf.readInt(state) - val nameHash = raf.readInt(state) - } + } // BTreeNode - // Type 9 Record Layout - Creation Order for Indexed Attributes - inner class Record9(state: OpenFileState) { - val heapId: ByteArray = raf.readByteArray(state, 8) - val flags = raf.readByte(state) - val creationOrder = raf.readInt(state) + fun readRecord(state: OpenFileState, type: Int): ChunkImpl { + return when (type) { + 10 -> readRecord10(state, chunkShape.toIntArray(), chunkSize.toInt()) + 11 -> readRecord11(state, chunkShape.toIntArray() ) + else -> throw IllegalStateException() + } } // Type 10 Record Layout - Non-filtered Dataset Chunks - inner class Record10(state: OpenFileState, dims : IntArray, chunkSize: Int) { - val chunk : ChunkImpl + fun readRecord10(state: OpenFileState, dims : IntArray, chunkSize: Int): ChunkImpl { + val address = raf.readOffset(state) - init { - val address = h5.readOffset(state) - - // This field is the scaled offset of the chunk within the dataset. n is the number of dimensions for the dataset. - val scaledOffset = LongArray(dims.size) { raf.readLong(state) } - - // Scaled offset is calculated by dividing the chunk dimension sizes into the chunk offsets. - // so to get the chunk offset: - // jhdf - // int[] chunkOffset = new int[datasetInfo.getDatasetDimensions().length]; - // for (int i = 0; i < chunkOffset.length; i++) { - // chunkOffset[i] = Utils.readBytesAsUnsignedInt(buffer, 8) * datasetInfo.getChunkDimensions()[i]; - // } - val chunkOffset = scaledOffset.mapIndexed { idx, scaledOffset -> (scaledOffset * dims[idx]).toInt() } - - // ChunkImpl(val address: Long, val size: Int, val chunkOffset: IntArray, val filterMask: Int?) - chunk = ChunkImpl(address, chunkSize, chunkOffset.toIntArray(), null) - } + // This field is the scaled offset of the chunk within the dataset. n is the number of dimensions for the dataset. + val scaledOffset = LongArray(dims.size) { raf.readLong(state) } + + // Scaled offset is calculated by dividing the chunk dimension sizes into the chunk offsets. + // so to get the chunk offset: + // jhdf + // int[] chunkOffset = new int[datasetInfo.getDatasetDimensions().length]; + // for (int i = 0; i < chunkOffset.length; i++) { + // chunkOffset[i] = Utils.readBytesAsUnsignedInt(buffer, 8) * datasetInfo.getChunkDimensions()[i]; + // } + val chunkOffset = scaledOffset.mapIndexed { idx, scaledOffset -> (scaledOffset * dims[idx]).toInt() } + + return ChunkImpl(address, chunkSize, chunkOffset.toIntArray(), null, tiling) } // Type 11 Record Layout - Filtered Dataset Chunks - inner class Record11(state: OpenFileState, dims : IntArray) { - val chunk : ChunkImpl + fun readRecord11(state: OpenFileState, dims : IntArray): ChunkImpl { + val address = raf.readOffset(state) + + // LOOK variable size based on what? "Chunk Size (variable size; at most 8 bytes)" + // jhdf + // final int chunkSizeBytes = buffer.limit() + // - 8 // size of offsets + // - 4 // filter mask + // - datasetInfo.getDatasetDimensions().length * 8; // dimension offsets + val rank = dims.size + val chunkSizeBytes = recordSize - 8 - 4 - rank * 8 + val chunkSize = raf.readVariableSizeUnsigned(state, chunkSizeBytes).toInt() - init { - val address = h5.readOffset(state) - - // LOOK variable size based on what? "Chunk Size (variable size; at most 8 bytes)" - // jhdf - // final int chunkSizeBytes = buffer.limit() - // - 8 // size of offsets - // - 4 // filter mask - // - datasetInfo.getDatasetDimensions().length * 8; // dimension offsets - val rank = dims.size - val chunkSizeBytes = recordSize - 8 - 4 - rank * 8 - val chunkSize = h5.readVariableSizeUnsigned(state, chunkSizeBytes).toInt() - - val filterMask = raf.readInt(state) - - // This field is the scaled offset of the chunk within the dataset. n is the number of dimensions for the dataset. - val scaledOffset = LongArray(rank) { raf.readLong(state) } - - // Scaled offset is calculated by dividing the chunk dimension sizes into the chunk offsets. - // so to get the chunk offset: - // jhdf - // int[] chunkOffset = new int[datasetInfo.getDatasetDimensions().length]; - // for (int i = 0; i < chunkOffset.length; i++) { - // chunkOffset[i] = Utils.readBytesAsUnsignedInt(buffer, 8) * datasetInfo.getChunkDimensions()[i]; - // } - val chunkOffset = scaledOffset.mapIndexed { idx, scaledOffset -> (scaledOffset * dims[idx]).toInt() } - - // ChunkImpl(val address: Long, val size: Int, val chunkOffset: IntArray, val filterMask: Int?) - chunk = ChunkImpl(address, chunkSize, chunkOffset.toIntArray(), filterMask) - } - } + val filterMask = raf.readInt(state) - // TODO this is probably not handling missing chunks correctly. See BTree1data, which iterates over tiles. - fun chunkIterator() : Iterator = ChunkIterator() + // This field is the scaled offset of the chunk within the dataset. n is the number of dimensions for the dataset. + val scaledOffset = LongArray(rank) { raf.readLong(state) } - private inner class ChunkIterator : AbstractIterator() { - var count = 0 + // Scaled offset is calculated by dividing the chunk dimension sizes into the chunk offsets. + // so to get the chunk offset: + // jhdf + // int[] chunkOffset = new int[datasetInfo.getDatasetDimensions().length]; + // for (int i = 0; i < chunkOffset.length; i++) { + // chunkOffset[i] = Utils.readBytesAsUnsignedInt(buffer, 8) * datasetInfo.getChunkDimensions()[i]; + // } + val chunkOffset = scaledOffset.mapIndexed { idx, scaledOffset -> (scaledOffset * dims[idx]).toInt() } - override fun computeNext() { - if (count >= records.size) { - return done() - } - val chunk = when (btreeType) { - 10 -> (records[count] as Record10).chunk - 11 -> (records[count] as Record11).chunk - else -> throw RuntimeException() - } - setNext(chunk) - count++ - } + // ChunkImpl(val address: Long, val size: Int, val chunkOffset: IntArray, val filterMask: Int?) + return ChunkImpl(address, chunkSize, chunkOffset.toIntArray(), filterMask, tiling) } - companion object { - internal fun findRecord1byId(records: List, hugeObjectID: Int): Record1? { - for (record in records) { - if (record is Record1 && record.hugeObjectID == hugeObjectID.toLong()) return record - } - return null - } + fun missingDataChunk(order: Int) : ChunkImpl { + return ChunkImpl(-1, 0, tiling.orderToIndex(order.toLong()).toIntArray(), 0, tiling) } + } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ChunkedDataLayoutMessageV4.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ChunkedDataLayoutV4.kt similarity index 93% rename from core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ChunkedDataLayoutMessageV4.kt rename to core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ChunkedDataLayoutV4.kt index 78153d6c..765d33dd 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ChunkedDataLayoutMessageV4.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/ChunkedDataLayoutV4.kt @@ -3,8 +3,10 @@ package com.sunya.netchdf.hdf5 import com.sunya.cdm.api.computeSize +import com.sunya.cdm.api.toLongArray import com.sunya.cdm.iosp.OpenFileIF import com.sunya.cdm.iosp.OpenFileState +import com.sunya.cdm.layout.Tiling import com.sunya.cdm.util.InternalLibraryApi import io.github.oshai.kotlinlogging.KotlinLogging import kotlin.math.ceil @@ -13,7 +15,7 @@ import kotlin.math.ceil // jhdf @OptIn(InternalLibraryApi::class) -internal fun readChunkedDataLayoutMessageV4(builder: H5builder, raf: OpenFileIF, state : OpenFileState) : DataLayoutMessage { +internal fun readChunkedDataLayoutV4(builder: H5builder, raf: OpenFileIF, state : OpenFileState) : DataLayoutMessage { val version = raf.readByte(state) val layoutClass = raf.readByte(state) val flags = raf.readByte(state) @@ -273,8 +275,26 @@ fun chunkIndexToChunkOffset(chunkIndex: Int, chunkDimensions: IntArray, datasetD } //////////////////////////////////////////////////// -data class ChunkImpl(val address: Long, val size: Int, val chunkOffset: IntArray, val filterMask: Int?) { +data class ChunkImpl(val address: Long, val size: Int, val chunkOffset: IntArray, val filterMask: Int?, val tiling: Tiling?=null): DataChunkIF { override fun toString(): String { return "ChunkImpl(address=$address, size=$size, chunkOffset=${chunkOffset.contentToString()}, filterMask=$filterMask)" } + + override fun childAddress() = address + + override fun offsets() = chunkOffset.toLongArray() + + override fun isMissing() = address <= 0 + + override fun chunkSize() = size + + override fun filterMask() = filterMask ?: 0 + + override fun show(): String { + return if (tiling != null) { + "address=$address, chunkSize=${size}, chunkStart=${offsets().contentToString()}, tile= ${tiling.tile(offsets() ).contentToString()}" + } else { + "TODO(Not yet implemented)" + } + } } \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/DataChunkSequence.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/DataChunkSequence.kt new file mode 100644 index 00000000..c20834e1 --- /dev/null +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/DataChunkSequence.kt @@ -0,0 +1,15 @@ +package com.sunya.netchdf.hdf5 + +interface DataChunkSequence { + fun asSequence(): Sequence +} + +interface DataChunkIF { + fun childAddress(): Long + fun offsets(): LongArray + fun isMissing(): Boolean + fun chunkSize(): Int + fun filterMask(): Int + + fun show(): String +} \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt index 6d8c5636..f6a9105e 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt @@ -174,12 +174,12 @@ internal class FractalHeap(private val h5: H5builder, forWho: String, address: L when (subtype) { 1, 2 -> { if (btreeHugeObjects == null) { // lazy - val local = BTree2data(h5, "FractalHeap btreeHugeObjects", btreeAddressHugeObjects) + val local = BTree2(h5.makeFileExtended(), "FractalHeap btreeHugeObjects", btreeAddressHugeObjects) require(local.btreeType == subtype) btreeHugeObjects = local.records } - val record1: BTree2data.Record1? = BTree2data.findRecord1byId(btreeHugeObjects!!, offset) + val record1: BTree2.Record1? = BTree2.findRecord1byId(btreeHugeObjects!!, offset) if (record1 == null) { throw RuntimeException("Cant find DHeapId=$offset") } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt index 0687ea23..a7798634 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5builder.kt @@ -407,11 +407,15 @@ class H5builder( return Attribute(att.name, Datatype.STRING, svalues) } - fun openFileExtended(): OpenFileExtended { + fun openNewFileExtended(): OpenFileExtended { val raf: OpenFileIF = OkioFile(this.raf.location()) return OpenFileExtended(raf, this.isLengthLong, this.isOffsetLong, this.superblockStart) } + fun makeFileExtended(): OpenFileExtended { + return OpenFileExtended(this.raf, this.isLengthLong, this.isOffsetLong, this.superblockStart) + } + companion object { // special attribute names in HDF5 const val HDF5_CLASS = "CLASS" diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkConcurrent.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkConcurrent.kt index 707110e6..13564728 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkConcurrent.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkConcurrent.kt @@ -7,14 +7,12 @@ import com.sunya.cdm.api.Datatype import com.sunya.cdm.api.Section import com.sunya.cdm.api.SectionPartial import com.sunya.cdm.api.Variable -import com.sunya.cdm.api.computeSize import com.sunya.cdm.api.toIntArray import com.sunya.cdm.api.toLongArray import com.sunya.cdm.array.ArrayTyped import com.sunya.cdm.iosp.OpenFileState import com.sunya.cdm.layout.Chunker import com.sunya.cdm.layout.IndexSpace -import com.sunya.cdm.layout.Tiling import com.sunya.cdm.layout.transferMissingNelems import com.sunya.cdm.util.InternalLibraryApi import kotlinx.coroutines.CoroutineScope @@ -29,32 +27,26 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.yield @ExperimentalCoroutinesApi -class H5chunkConcurrent(val h5: H5builder, val v2: Variable, wantSection: SectionPartial?) { - val rafext: OpenFileExtended = h5.openFileExtended() - internal val bTree: BTree1data +class H5chunkConcurrent(val h5: H5builder, val v2: Variable, wantSection: SectionPartial?, ) { + val rafext: OpenFileExtended = h5.makeFileExtended() val varShape = v2.shape - val chunkShape: IntArray - val tiling: Tiling - val nchunks: Long val wantSpace: IndexSpace val allData : Boolean + val chunks: DataChunkSequence init { val useSection = SectionPartial.fill(wantSection, v2.shape) wantSpace = IndexSpace(useSection) allData = (wantSection == null) || (useSection == Section(varShape)) - require(v2.spObject is DataContainerVariable) - val vinfo = v2.spObject - require(vinfo.mdl is DataLayoutBTreeVer1) - val mdl = vinfo.mdl - chunkShape = mdl.chunkDims - tiling = Tiling(varShape, chunkShape.toLongArray()) - nchunks = tiling.tileShape.computeSize() - - // its not obvious you actually need a seperate raf - bTree = BTree1data(rafext, mdl.btreeAddress, varShape, chunkShape.toLongArray()) + val vinfo = v2.spObject as DataContainerVariable + if (vinfo.mdl is DataLayoutBTreeVer1) { + val mdl = vinfo.mdl + chunks = BTree1data(rafext, mdl.btreeAddress, varShape, mdl.chunkDims.toLongArray()) + } else { + throw RuntimeException() + } } fun readChunks(nthreads: Int, lamda: (ArraySection) -> Unit, done: () -> Unit) { @@ -62,7 +54,7 @@ class H5chunkConcurrent(val h5: H5builder, val v2: Variable, wantSection: runBlocking { val jobs = mutableListOf() val workers = mutableListOf() - val chunkProducer = produceChunks(bTree.asSequence()) + val chunkProducer = produceChunks(chunks.asSequence()) repeat(nthreads) { val worker = Worker() jobs.add( launchJob(worker, chunkProducer, lamda)) @@ -73,12 +65,11 @@ class H5chunkConcurrent(val h5: H5builder, val v2: Variable, wantSection: joinAll(*jobs.toTypedArray()) workers.forEach { it.rafext.close() } } - rafext.close() done() } private var count = 0 - private fun CoroutineScope.produceChunks(producer: Sequence>): ReceiveChannel> = + private fun CoroutineScope.produceChunks(producer: Sequence): ReceiveChannel = produce { for (dataChunk in producer) { send(dataChunk) @@ -90,18 +81,18 @@ class H5chunkConcurrent(val h5: H5builder, val v2: Variable, wantSection: private fun CoroutineScope.launchJob( worker: Worker, - input: ReceiveChannel>, + input: ReceiveChannel, lamda: (ArraySection) -> Unit, ) = launch(Dispatchers.Default) { - for (pair: Pair in input) { - val arraySection = worker.work(pair.second) + for (chunk: DataChunkIF in input) { + val arraySection = worker.work(chunk) if (arraySection != null) lamda(arraySection) yield() } } private inner class Worker() { - val rafext: OpenFileExtended = h5.openFileExtended() // here we need a seperate raf + val rafext: OpenFileExtended = h5.openNewFileExtended() // here we need a seperate raf val vinfo: DataContainerVariable = v2.spObject as DataContainerVariable val h5type: H5TypeInfo @@ -129,14 +120,14 @@ class H5chunkConcurrent(val h5: H5builder, val v2: Variable, wantSection: val intersectSpace = if (useEntireChunk) dataSpace else wantSpace.intersect(dataSpace) val ba = if (dataChunk.isMissing()) { - if (debugChunking) println(" missing ${dataChunk.show(tiling)}") + if (debugChunking) println(" missing ${dataChunk.show()}") val sizeBytes = intersectSpace.totalElements * elemSize val bbmissing = ByteArray(sizeBytes.toInt()) transferMissingNelems(vinfo.fillValue, intersectSpace.totalElements.toInt(), bbmissing, 0) if (debugChunking) println(" missing transfer ${intersectSpace.totalElements} fillValue=${vinfo.fillValue}") bbmissing } else { - if (debugChunking) println(" chunkIterator=${dataChunk.show(tiling)}") + if (debugChunking) println(" chunkIterator=${dataChunk.show()}") state.pos = dataChunk.childAddress() val rawdata = rafext.readByteArray(state, dataChunk.chunkSize()) val filteredData = if (dataChunk.filterMask() == null) rawdata else filters.apply(rawdata, dataChunk.filterMask()!!) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt index be0f5840..abfa58a4 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt @@ -14,7 +14,7 @@ import kotlin.collections.iterator private val debugChunking = false // DataLayoutSingleChunk4, DataLayoutImplicit4, DataLayoutFixedArray4, DataLayoutExtensibleArray4, DataLayoutBtreeVer2 -internal fun H5builder.readChunkedData(v2: Variable, wantSection: Section, index: Iterator): ArrayTyped { +internal fun H5builder.readChunkedData(v2: Variable, wantSection: Section, index: Iterator): ArrayTyped { val vinfo = v2.spObject as DataContainerVariable val h5type = vinfo.h5type @@ -34,22 +34,23 @@ internal fun H5builder.readChunkedData(v2: Variable, wantSection: Section val filters = FilterPipeline(v2.name, vinfo.mfp, vinfo.h5type.isBE) val state = OpenFileState(0L, vinfo.h5type.isBE) - // just run through all the chunks, we wont read any that we dont want - for (dataChunk : ChunkImpl in index) { - val dataSection = IndexSpace(v2.rank, dataChunk.chunkOffset.toLongArray(), vinfo.storageDims) + // just run through all the chunks, we wont read any that we dont need + for (dataChunk: DataChunkIF in index) { + val dataSection = IndexSpace(v2.rank, dataChunk.offsets(), vinfo.storageDims) val chunker = Chunker(dataSection, wantSpace) // each DataChunkEntry has its own Chunker iteration - if (chunker.nelems > 0) { // TODO efficient enough ?? - /* if (dataChunk.isMissing()) { - if (debugChunking) println(" missing ${dataChunk.show(tiledData.tiling)}") - chunker.transferMissing(vinfo.fillValue, elemSize, ba) - } else { */ - state.pos = dataChunk.address - val rawdata = this.raf.readByteArray(state, dataChunk.size) - val filteredData = if (vinfo.mfp == null || dataChunk.filterMask == null) rawdata - else filters.apply(rawdata, dataChunk.filterMask) - chunker.transferBA(filteredData, 0, elemSize, ba, 0) + if (chunker.nelems > 0) { + if (dataChunk.isMissing()) { + if (debugChunking) println(" missing ${dataChunk.show()}") + chunker.transferMissing(vinfo.fillValue, elemSize, ba) + } else { + // println(dataChunk.show()) + state.pos = dataChunk.childAddress() + val rawdata = this.raf.readByteArray(state, dataChunk.chunkSize()) + val filteredData = if (vinfo.mfp == null || dataChunk.filterMask() == null) rawdata + else filters.apply(rawdata, dataChunk.filterMask()) + chunker.transferBA(filteredData, 0, elemSize, ba, 0) + } } - // } } val shape = wantSpace.shape.toIntArray() @@ -142,7 +143,7 @@ internal fun H5builder.readBtree1data(v2: Variable, wantSection: Section) // varShape: LongArray, // chunkShape: LongArray, //) - val rafext: OpenFileExtended = this.openFileExtended() + val rafext: OpenFileExtended = this.openNewFileExtended() BTree1data(rafext, vinfo.dataPos, v2.shape, vinfo.storageDims) } else { throw RuntimeException("Unsupported mdl ${vinfo.mdl}") @@ -154,7 +155,7 @@ internal fun H5builder.readBtree1data(v2: Variable, wantSection: Section) var transferChunks = 0 val state = OpenFileState(0L, vinfo.h5type.isBE) - btree1.asSequence().forEach { (order, dataChunk) -> + btree1.asSequence().forEach { dataChunk -> val dataSection = IndexSpace(v2.rank, dataChunk.offsets(), vinfo.storageDims) val chunker = Chunker(dataSection, wantSpace) // each DataChunkEntry has its own Chunker iteration if (dataChunk.isMissing()) { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt index 6c7d8536..b6854a12 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt @@ -59,12 +59,12 @@ internal fun H5builder.readGroupNew( check(btreeAddress >= 0) { "no valid btree for GroupNew with Fractal Heap" } // read in btree and all entries - val btree2j = BTree2data(this, parent.name, btreeAddress) + val btree2j = BTree2(this.makeFileExtended(), parent.name, btreeAddress) for (record in btree2j.records) { val heapId: ByteArray = when (btree2j.btreeType) { - 5 -> (record as BTree2data.Record5).heapId - 6 -> (record as BTree2data.Record6).heapId - else -> throw RuntimeException("btree2 type ${btree2j.btreeType} mot supported") + 5 -> (record as BTree2.Record5).heapId + 6 -> (record as BTree2.Record6).heapId + else -> throw RuntimeException("btree2 type ${btree2j.btreeType} not supported") } // the heapId points to a Link message in the Fractal Heap diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt index 89ea016f..84995f63 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt @@ -110,7 +110,7 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { } else if (vinfo.mdl is DataLayoutBtreeVer2) { // header.readBtreeVer2j(v2, wantSection) - val index = BTree2data(header, v2.name, vinfo.dataPos, vinfo.storageDims) + val index = BTree2data(header.makeFileExtended(), v2.name, vinfo.dataPos, v2.shape, vinfo.storageDims) header.readChunkedData(v2, section, index.chunkIterator()) } else { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt index 25585378..747b1e6c 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt @@ -130,7 +130,7 @@ internal fun H5builder.readDataLayoutMessage(state : OpenFileState) : DataLayout } // version 4, layoutClass = 2 is too complex for structdls - return readChunkedDataLayoutMessageV4(this, raf, state) + return readChunkedDataLayoutV4(this, raf, state) } throw RuntimeException() } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt index 25c44fbb..c5e3367f 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt @@ -884,14 +884,14 @@ private fun H5builder.readAttributesFromInfoMessage2( val btreeAddress: Long = attributeOrderBtreeAddress ?: attributeNameBtreeAddress if (btreeAddress < 0 || fractalHeapAddress < 0) return emptyList() - val btree2j = BTree2data(this, "AttributeInfoMessage", btreeAddress) + val btree2j = BTree2(this.makeFileExtended(), "AttributeInfoMessage", btreeAddress) val fractalHeapj = FractalHeap(this, "AttributeInfoMessage", fractalHeapAddress) val attMessages = mutableListOf() for (record in btree2j.records) { val heapId: ByteArray = when (btree2j.btreeType) { - 8 -> (record as BTree2data.Record8).heapId - 9 -> (record as BTree2data.Record9).heapId + 8 -> (record as BTree2.Record8).heapId + 9 -> (record as BTree2.Record9).heapId else -> continue } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/OpenFileExtended.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/OpenFileExtended.kt index 95ac5b41..3f4ef486 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/OpenFileExtended.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/OpenFileExtended.kt @@ -17,6 +17,10 @@ class OpenFileExtended(val delegate: OpenFileIF, return if (isOffsetLong) delegate.readLong(state) else delegate.readInt(state).toLong() } + fun sizeOffsets(): Int { + return if (isOffsetLong) 8 else 4 + } + fun getFileOffset(address: Long): Long { return startingOffset + address } @@ -25,5 +29,44 @@ class OpenFileExtended(val delegate: OpenFileIF, return getFileOffset(readOffset(state)) } + fun readVariableSizeUnsigned(state : OpenFileState, size: Int): Long { + val vv: Long + when (size) { + 1 -> vv = delegate.readByte(state).toUByte().toLong() + 2 -> vv = delegate.readShort(state).toUShort().toLong() + 4 -> vv = delegate.readInt(state).toUInt().toLong() + 8 -> vv = delegate.readLong(state) + else -> vv = readVariableSizeN(state, size) + } + return vv + } + + fun readVariableSizeDimension(state : OpenFileState, size: Byte): Int { + val vv: Int + val sizeInt = size.toInt() + when (sizeInt) { + 1 -> vv = delegate.readByte(state).toUByte().toInt() + 2 -> vv = delegate.readShort(state).toUShort().toInt() + 4 -> vv = delegate.readInt(state).toUInt().toInt() + else -> { + val vs = readVariableSizeN(state, sizeInt) + vv = vs.toInt() + } + } + return vv + } + + private fun readVariableSizeN(state : OpenFileState, nbytes : Int): Long { + val ch = IntArray(nbytes) + for (i in 0 until nbytes) ch[i] = delegate.readByte(state).toInt() + var result = ch[nbytes - 1].toLong() + for (i in nbytes - 2 downTo 0) { + result = result shl 8 + result += ch[i].toLong() + } + return result + } + + override fun close() = delegate.close() } \ No newline at end of file diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1dataTest.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1dataTest.kt index 7e57375e..d890cd63 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1dataTest.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1dataTest.kt @@ -28,7 +28,7 @@ class Btree1dataTest { ?: throw RuntimeException("cant find $varname") println(" ${myvar.nameAndShape()}") - val rafext: OpenFileExtended = h5.openFileExtended() + val rafext: OpenFileExtended = h5.openNewFileExtended() val varShape = myvar.shape require(myvar.spObject is DataContainerVariable) @@ -41,7 +41,7 @@ class Btree1dataTest { val bTreeExt = BTree1data(rafext, mdl.btreeAddress, varShape, chunkShape.toLongArray()) // val rootNode = bTreeExt.rootNode() - bTreeExt.asSequence().forEach { (key, value) -> println("Key: ${key}, Value: ${value.show()}") } + bTreeExt.asSequence().forEach { value -> println("Value: ${value.show()}") } } } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt index fdf0149f..10bb7272 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt @@ -19,7 +19,8 @@ class NetchdfClibTest { companion object { fun files(): Iterator { - return sequenceOf(N3Files.files().asSequence(), + return sequenceOf( + N3Files.files().asSequence(), N4Files.files().asSequence(), H5Files.files().asSequence(), H4Files.files().asSequence(), diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt index 268d2613..2e6e54d3 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt @@ -113,7 +113,7 @@ class CountVersions { if (ncfile == null) { println("Not a netchdf file=$filename ") } else { - val hdf5File = if (ncfile.type() in listOf("netcdf4", "hdf")) { + val hdf5File = if (ncfile.type() in listOf("netcdf4", "hdf5")) { ncfile as Hdf5File } else null @@ -156,7 +156,7 @@ class CountVersions { if (ncfile == null) { println("Not a netchdf file=$filename ") } else { - if (ncfile.type() in listOf("netcdf4", "hdf")) { + if (ncfile.type() in listOf("netcdf4", "hdf5")) { val hdf5File = ncfile as Hdf5File ncfile.rootGroup().allVariables().forEach { v -> val layout = hdf5File.layoutName(v) @@ -179,5 +179,40 @@ class CountVersions { sorted.keys.forEach{ println("${sorted[it]} == '$it'") } } + @Test + fun showLayoutTypes() { + fun h5files(): Iterator { + return sequenceOf( + //N4Files.Companion.files().asSequence(), + //H5Files.Companion.files().asSequence(), + //NetchdfExtraFiles.Companion.files(false).asSequence(), + JhdfFiles.Companion.files().asSequence(), + ) + .flatten() + .iterator() + } + + h5files().forEach { filename -> + try { + openNetchdfFile(filename).use { ncfile -> + if (ncfile == null) { + println("Not a netchdf file=$filename ") + } else { + val filetype = ncfile.type() + if (filetype in listOf("netcdf4", "hdf5")) { + val hdf5File = ncfile as Hdf5File + ncfile.rootGroup().allVariables().forEach { v -> + val layout = hdf5File.layoutName(v) + println(" ${layout} ${v.nameAndShape()} ($filename)") + } + } + } + } + } catch (e: Throwable) { + e.printStackTrace() + } + } + } + data class LayoutCount(var count: Int = 0, var size: Long = 0) } \ No newline at end of file diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt index 32a0da0d..2098ca3a 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt @@ -39,6 +39,11 @@ class JhdfCompare { // compareDataWithJhdf(filename, showData = false, showCdl = true) } + @Test + fun testBtreeVer4() { + compareDataWithJhdf("../core/src/commonTest/data/jhdf/chunked_v4_datasets.hdf5", "/btree_v2/large_int16", true, true) + } + // @Test horror show fun superblocks() { compareDataWithJhdf(testData + "netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_8/nc_test4__tst_xplatform2_3.nc", null, true, true)