From c1cde0ed4452fa87c16bc959c71c4504e877fe8e Mon Sep 17 00:00:00 2001 From: JohnLCaron Date: Sat, 26 Jul 2025 06:19:29 -0600 Subject: [PATCH] Remove BTree1, H5TiledData1, H5chunkIterator. minor cleanup --- cli/build.gradle.kts | 4 +- .../kotlin/com/sunya/cdm/api/Variable.kt | 4 + .../com/sunya/cdm/array/StructureMember.kt | 2 + .../kotlin/com/sunya/netchdf/hdf4/Hdf4File.kt | 8 +- .../kotlin/com/sunya/netchdf/hdf5/BTree1.kt | 4 +- .../com/sunya/netchdf/hdf5/BTree1data.kt | 2 +- .../com/sunya/netchdf/hdf5/BTree2data.kt | 1 + .../com/sunya/netchdf/hdf5/H5TiledData1.kt | 4 +- .../com/sunya/netchdf/hdf5/H5cdmBuilder.kt | 5 + .../com/sunya/netchdf/hdf5/H5chunkIterator.kt | 6 +- .../com/sunya/netchdf/hdf5/H5chunkReader.kt | 95 ++++++++++++------- .../kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt | 43 +++++---- .../com/sunya/netchdf/hdf5/MessageHeader.kt | 2 +- .../com/sunya/netchdf/hdf5/StructDsl.kt | 12 +-- .../com/sunya/netchdf/netcdf3/Netcdf3File.kt | 10 +- ...{Hdf5Compare.kt => Hdf5CompareWithClib.kt} | 6 +- .../netchdf/hdf5/H5readConcurrentTest.kt | 11 ++- 17 files changed, 131 insertions(+), 88 deletions(-) rename testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/{Hdf5Compare.kt => Hdf5CompareWithClib.kt} (97%) diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index f0733ce2..521332e1 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -3,7 +3,7 @@ } dependencies { - implementation(project(":core")) + api(project(":core")) implementation(libs.lzf) implementation(libs.lz4) @@ -33,4 +33,4 @@ tasks.register("uberJar") { }) } -project.tasks["compileJava"].dependsOn(":core:allMetadataJar") +// project.tasks["compileJava"].dependsOn(":core:allMetadataJar") diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt b/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt index d94f0476..89ecba00 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt @@ -51,6 +51,10 @@ data class Variable( return "$datatype ${fullname()}${shape.contentToString()}" } + override fun toString(): String { + return "Variable(${nameAndShape()}, group=${group.fullname()}, nelems=$nelems, spObject=$spObject)" + } + @InternalLibraryApi class Builder(val name : String, val datatype : Datatype) { val dimensions = mutableListOf() diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt b/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt index 5ca075b5..fa5192ed 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt @@ -1,3 +1,5 @@ +@file:OptIn(ExperimentalUnsignedTypes::class) + package com.sunya.cdm.array import com.sunya.cdm.api.Datatype diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Hdf4File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Hdf4File.kt index e51da22c..d59db4fc 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Hdf4File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Hdf4File.kt @@ -46,17 +46,17 @@ class Hdf4File(val filename : String) : Netchdf { } } - override fun chunkIterator(v2: Variable, section: SectionPartial?, maxElements : Int?): Iterator> { + override fun chunkIterator(v2: Variable, wantSection: SectionPartial?, maxElements : Int?): Iterator> { if (v2.nelems == 0L) { return listOf>().iterator() } - val wantSection = SectionPartial.fill(section, v2.shape) + val section = SectionPartial.fill(wantSection, v2.shape) val vinfo = v2.spObject as Vinfo return if (vinfo.isChunked) { // LOOK isLinked? - H4chunkIterator(header, v2, wantSection) + H4chunkIterator(header, v2, section) } else { - H4maxIterator(v2, wantSection, maxElements ?: 100_000) + H4maxIterator(v2, section, maxElements ?: 100_000) } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt index f3882047..5bd1f9aa 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1.kt @@ -6,7 +6,7 @@ import com.sunya.cdm.iosp.OpenFileState import com.sunya.cdm.layout.Tiling import com.sunya.cdm.util.InternalLibraryApi -/** B-tree, version 1, used for data (node type 1) */ +/** B-tree, version 1, used for data (node type 1) internal class BTree1( val h5: H5builder, val rootNodeAddress: Long, @@ -130,7 +130,7 @@ internal class BTree1( ", tile= ${tiling.tile(key.offsets).contentToString()} idx=$idx" } -} +} */ interface DataChunkIF { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt index 4f115c02..6d22d00c 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt @@ -118,7 +118,7 @@ internal class BTree1data( override fun chunkSize() = key.chunkSize override fun filterMask() = key.filterMask - override fun show(tiling : Tiling) : String = "chunkSize=${key.chunkSize}, chunkStart=${offsets().contentToString()}" + + override fun show(tiling : Tiling) : String = "order=$key, chunkSize=${key.chunkSize}, chunkStart=${offsets().contentToString()}" + ", tile= ${tiling.tile(offsets() ).contentToString()}" fun show() = show(tiling) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt index 90495989..269c4669 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt @@ -306,6 +306,7 @@ internal class BTree2data(private val h5: H5builder, owner: String, address: Lon } } + // TODO this is probably not handling missing chunks correctly. See BTree1data, which iterates over tiles. fun chunkIterator() : Iterator = ChunkIterator() private inner class ChunkIterator : AbstractIterator() { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt index 3cf97b3a..c750adc1 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5TiledData1.kt @@ -4,7 +4,7 @@ import com.sunya.cdm.layout.IndexSpace import com.sunya.cdm.layout.IndexND import com.sunya.cdm.layout.Tiling -/** wraps BTree1 to handle iterating through tiled data (aka chunked data) */ +/** wraps BTree1 to handle iterating through tiled data (aka chunked data) internal class H5TiledData1(val btree : BTree1, val varShape: LongArray, val chunkShape: LongArray) { private val check = true private val debug = false @@ -104,4 +104,4 @@ internal class H5TiledData1(val btree : BTree1, val varShape: LongArray, val chu return "TiledData(chunk=${chunkShape.contentToString()}, readHit=$readHit, readMiss=$readMiss)" } -} \ No newline at end of file +} */ \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt index e38fa22b..071edcd4 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt @@ -280,6 +280,11 @@ internal class DataContainerVariable( else -> throw RuntimeException() } } + + override fun toString(): String { + return "DataContainerVariable(mdl=$mdl, mfp=$mfp, onlyFillValue=$onlyFillValue)" + } + } internal fun getFillValue(h5 : H5builder, v5 : H5Variable, h5type: H5TypeInfo): ByteArray { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt index 03571e81..44e6def2 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkIterator.kt @@ -8,9 +8,7 @@ import com.sunya.cdm.layout.IndexSpace import com.sunya.cdm.layout.transferMissingNelems import com.sunya.cdm.util.InternalLibraryApi -// TODO assumes BTree1, could it include BTree2? any chunked reader ? -// only used in Netchdf.readChunksConcurrent - +/* (to be removed) @OptIn(InternalLibraryApi::class) internal class H5chunkIterator(val h5 : H5builder, val v2: Variable, val wantSection : Section) : AbstractIterator>() { private val debugChunking = false @@ -86,5 +84,5 @@ internal class H5chunkIterator(val h5 : H5builder, val v2: Variable, val w return ArraySection(array, intersectSpace.section(v2.shape)) // LOOK use space instead of Section ?? } -} +} */ diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt index c1dac0de..be0f5840 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt @@ -2,7 +2,6 @@ package com.sunya.netchdf.hdf5 -import com.fleeksoft.charset.decodeToString import com.sunya.cdm.api.* import com.sunya.cdm.array.* import com.sunya.cdm.iosp.OpenFileState @@ -62,8 +61,8 @@ internal fun H5builder.readChunkedData(v2: Variable, wantSection: Section } } -/* TODO can we use concurrent reading ?? -internal fun readBtree1data(v2: Variable, wantSection: Section): ArrayTyped { +/* DataLayoutBTreeVer1 (to be removed) +internal fun H5builder.readBtreeVer1(v2: Variable, wantSection: Section): ArrayTyped { val vinfo = v2.spObject as DataContainerVariable val h5type = vinfo.h5type @@ -77,33 +76,45 @@ internal fun readBtree1data(v2: Variable, wantSection: Section): ArrayTyp } val ba = ByteArray(sizeBytes.toInt()) - val reader = H5chunkConcurrent(h5, v2, wantSection) - val availableProcessors = com.sunya.netchdf.util.getAvailableProcessors() - reader.readChunks(availableProcessors, lamda = { asection: ArraySection<*> -> - val (array, section) = asection - println(" section = ${section}") - val dataSpace = IndexSpace(section) - - val useEntireChunk = wantSpace.contains(dataSpace) - val intersectSpace = if (useEntireChunk) dataSpace else wantSpace.intersect(dataSpace) + val btree1 = if (vinfo.mdl is DataLayoutBTreeVer1) + BTree1(this, vinfo.dataPos, 1, vinfo.storageDims.size) + else + throw RuntimeException("Unsupprted mdl ${vinfo.mdl}") - val chunker = Chunker(dataSpace, wantSpace) // each DataChunkEntry has its own Chunker iteration - chunker.transferBA(array, 0, elemSize, ba, 0) + val tiledData = H5TiledData1(btree1, v2.shape, vinfo.storageDims) + val filters = FilterPipeline(v2.name, vinfo.mfp, vinfo.h5type.isBE) + if (debugChunking) println(" readChunkedData tiles=${tiledData.tiling}") - if (h5type.datatype5 == Datatype5.Vlen) { - // internal fun H5builder.processVlenIntoArray(h5type: H5TypeInfo, shape: IntArray, ba: ByteArray, nelems: Int, elemSize : Int): ArrayTyped { - this.processVlenIntoArray(h5type, intersectSpace.shape.toIntArray(), ba, intersectSpace.totalElements.toInt(), elemSize) + var transferChunks = 0 + val state = OpenFileState(0L, vinfo.h5type.isBE) + for (dataChunk: DataChunkIF in tiledData.dataChunks(wantSpace)) { // : Iterable + val dataSection = IndexSpace(v2.rank, dataChunk.offsets(), vinfo.storageDims) + val chunker = Chunker(dataSection, wantSpace) // each DataChunkEntry has its own Chunker iteration + if (dataChunk.isMissing()) { + if (debugChunking) println(" missing ${dataChunk.show(tiledData.tiling)}") + chunker.transferMissing(vinfo.fillValue, elemSize, ba) } else { - this.processDataIntoArray(ba, h5type.isBE, datatype, intersectSpace.shape.toIntArray(), h5type, elemSize) + if (debugChunking) println(" chunk=${dataChunk.show(tiledData.tiling)}") + state.pos = dataChunk.childAddress() + val chunkData = this.raf.readByteArray(state, dataChunk.chunkSize()) + val filteredData = if (dataChunk.filterMask() == null) chunkData + else filters.apply(chunkData, dataChunk.filterMask()!!) + chunker.transferBA(filteredData, 0, elemSize, ba, 0) + transferChunks += chunker.transferChunks } + } - }, done = { }) + val shape = wantSpace.shape.toIntArray() - return ArraySection(array, intersectSpace.section(v2.shape)) + return if (h5type.datatype5 == Datatype5.Vlen) { + this.processVlenIntoArray(h5type, shape, ba, wantSpace.totalElements.toInt(), elemSize) + } else { + this.processDataIntoArray(ba, vinfo.h5type.isBE, datatype, shape, h5type, elemSize) as ArrayTyped + } } */ // DataLayoutBTreeVer1 -internal fun H5builder.readBtreeVer1(v2: Variable, wantSection: Section): ArrayTyped { +internal fun H5builder.readBtree1data(v2: Variable, wantSection: Section): ArrayTyped { val vinfo = v2.spObject as DataContainerVariable val h5type = vinfo.h5type @@ -117,25 +128,40 @@ internal fun H5builder.readBtreeVer1(v2: Variable, wantSection: Section): } val ba = ByteArray(sizeBytes.toInt()) - val btree1 = if (vinfo.mdl is DataLayoutBTreeVer1) - BTree1(this, vinfo.dataPos, 1, vinfo.storageDims.size) - else - throw RuntimeException("Unsupprted mdl ${vinfo.mdl}") + val btree1 = if (vinfo.mdl is DataLayoutBTreeVer1) { + // internal class BTree1( + // val h5: H5builder, + // val rootNodeAddress: Long, + // val nodeType : Int, // 0 = group/symbol table, 1 = raw data chunks + // val ndimStorage: Int? = null // TODO allowed to be null ?? + //) + // BTree1(this, vinfo.dataPos, 1, vinfo.storageDims.size) + // internal class BTree1data( + // val raf: OpenFileExtended, + // rootNodeAddress: Long, + // varShape: LongArray, + // chunkShape: LongArray, + //) + val rafext: OpenFileExtended = this.openFileExtended() + BTree1data(rafext, vinfo.dataPos, v2.shape, vinfo.storageDims) + } else { + throw RuntimeException("Unsupported mdl ${vinfo.mdl}") + } - val tiledData = H5TiledData1(btree1, v2.shape, vinfo.storageDims) + //val tiledData = H5TiledData1(btree1, v2.shape, vinfo.storageDims) val filters = FilterPipeline(v2.name, vinfo.mfp, vinfo.h5type.isBE) - if (debugChunking) println(" readChunkedData tiles=${tiledData.tiling}") + //if (debugChunking) println(" readChunkedData tiles=${tiledData.tiling}") var transferChunks = 0 val state = OpenFileState(0L, vinfo.h5type.isBE) - for (dataChunk: DataChunkIF in tiledData.dataChunks(wantSpace)) { // : Iterable + btree1.asSequence().forEach { (order, dataChunk) -> val dataSection = IndexSpace(v2.rank, dataChunk.offsets(), vinfo.storageDims) val chunker = Chunker(dataSection, wantSpace) // each DataChunkEntry has its own Chunker iteration if (dataChunk.isMissing()) { - if (debugChunking) println(" missing ${dataChunk.show(tiledData.tiling)}") + if (debugChunking) println(" missing ${dataChunk.show()}") chunker.transferMissing(vinfo.fillValue, elemSize, ba) } else { - if (debugChunking) println(" chunk=${dataChunk.show(tiledData.tiling)}") + if (debugChunking) println(" chunk=${dataChunk.show()}") state.pos = dataChunk.childAddress() val chunkData = this.raf.readByteArray(state, dataChunk.chunkSize()) val filteredData = if (dataChunk.filterMask() == null) chunkData @@ -154,17 +180,18 @@ internal fun H5builder.readBtreeVer1(v2: Variable, wantSection: Section): } } -// DataLayoutBTreeVer1 -internal fun readBtree1data(hdf5: Hdf5File, v2: Variable, wantSection: SectionPartial?): ArrayTyped { +// DataLayoutBTreeVer1 using chunkIterator +internal fun readBtree1dataWithChunkIterator(hdf5: Hdf5File, v2: Variable, wantSection: SectionPartial?): ArrayTyped { val vinfo = v2.spObject as DataContainerVariable - val h5type = vinfo.h5type val datatype = vinfo.h5type.datatype() - val elemSize = vinfo.storageDims[vinfo.storageDims.size - 1].toInt() // last one is always the elements size val useSection = SectionPartial.fill(wantSection, v2.shape) val wantSpace = IndexSpace(useSection) val nelems = wantSpace.totalElements.toInt() + // we will be forever haunted by this + // val useDatatype = if (vinfo.h5type.datatype5 == Datatype5.String) Datatype.STRING else datatype + val values = when (datatype) { Datatype.BYTE -> ByteArray(nelems) Datatype.CHAR, Datatype.UBYTE, Datatype.ENUM1 -> UByteArray(nelems) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt index 2d80e7d6..89ea016f 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt @@ -13,12 +13,13 @@ import com.sunya.cdm.array.TypedByteArray import com.sunya.cdm.iosp.* import com.sunya.cdm.util.InternalLibraryApi import com.sunya.netchdf.util.Deque -import com.sunya.netchdf.util.useDefaultNThreads +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlin.String /** * @param strict true = make it agree with nclib if possible */ +@OptIn(ExperimentalCoroutinesApi::class) class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { private val raf : OpenFileIF = OkioFile(filename) val header : H5builder = H5builder(raf, strict) @@ -46,15 +47,15 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { return useNThreads ?: com.sunya.netchdf.util.useDefaultNThreads() } - override fun readArrayData(v2: Variable, section: SectionPartial?): ArrayTyped { + override fun readArrayData(v2: Variable, wantSection: SectionPartial?): ArrayTyped { if (v2.nelems == 0L) { return ArrayEmpty(v2.shape.toIntArray(), v2.datatype) } - val wantSection = SectionPartial.fill(section, v2.shape) + val section = SectionPartial.fill(wantSection, v2.shape) // promoted attributes if (v2.spObject is DataContainerAttribute) { - return header.readRegularData(v2.spObject, v2.datatype, wantSection) + return header.readRegularData(v2.spObject, v2.datatype, section) } val vinfo = v2.spObject as DataContainerVariable @@ -65,22 +66,24 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { return ArrayString(shapeMinus1, List(shapeMinus1.computeSize()) {""} ) as ArrayTyped } val tba = TypedByteArray(v2.datatype, vinfo.fillValue, 0, isBE = vinfo.h5type.isBE) - return ArraySingle(wantSection.shape.toIntArray(), v2.datatype, tba.get(0)) + return ArraySingle(section.shape.toIntArray(), v2.datatype, tba.get(0)) } return try { if (vinfo.mdl.isCompact) { val alldata = header.readCompactData(v2, v2.shape.toIntArray()) - alldata.section(wantSection) + alldata.section(section) } else if (vinfo.mdl.isContiguous) { - header.readRegularData(vinfo, v2.datatype, wantSection) + header.readRegularData(vinfo, v2.datatype, section) } else if (vinfo.mdl is DataLayoutBTreeVer1) { - if (v2.datatype == Datatype.COMPOUND) - header.readBtreeVer1(v2, wantSection) + // skip the concurrent read on the hard stuff + if (v2.datatype == Datatype.CHAR || v2.datatype == Datatype.COMPOUND || v2.datatype == Datatype.OPAQUE || + v2.datatype == Datatype.STRING || v2.datatype == Datatype.VLEN) + header.readBtree1data(v2, section) else - readBtree1data(this, v2, section) + readBtree1dataWithChunkIterator(this, v2, wantSection) } else if (vinfo.mdl is DataLayoutSingleChunk4) { // header.readSingleChunk(v2, wantSection) @@ -88,27 +91,27 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { val offset = IntArray(v2.rank) val chunk = ChunkImpl(vinfo.mdl.heapAddress, vinfo.mdl.chunkSize, offset, vinfo.mdl.filterMask) - header.readChunkedData(v2, wantSection, listOf(chunk).iterator()) + header.readChunkedData(v2, section, listOf(chunk).iterator()) } else if (vinfo.mdl is DataLayoutImplicit4) { // header.readImplicit4(v2, wantSection) val index = ImplicitChunkIndex(header, varShape=v2.shape.toIntArray(), vinfo.mdl) - header.readChunkedData(v2, wantSection, index.chunkIterator()) + header.readChunkedData(v2, section, index.chunkIterator()) } else if (vinfo.mdl is DataLayoutFixedArray4) { // header.readFixedArray4(v2, wantSection) val index = FixedArrayIndex(header, varShape=v2.shape.toIntArray(), vinfo.mdl) // mdl.fixedArrayIndex - header.readChunkedData(v2, wantSection, index.chunkIterator()) + header.readChunkedData(v2, section, index.chunkIterator()) } else if (vinfo.mdl is DataLayoutExtensibleArray4) { val index = ExtensibleArrayIndex(header, vinfo.mdl.indexAddress, v2.shape.toIntArray(), vinfo.mdl.chunkDimensions) - header.readChunkedData(v2, wantSection, index.chunkIterator()) + header.readChunkedData(v2, section, index.chunkIterator()) } else if (vinfo.mdl is DataLayoutBtreeVer2) { // header.readBtreeVer2j(v2, wantSection) val index = BTree2data(header, v2.name, vinfo.dataPos, vinfo.storageDims) - header.readChunkedData(v2, wantSection, index.chunkIterator()) + header.readChunkedData(v2, section, index.chunkIterator()) } else { throw RuntimeException("Unsupported data layer type ${vinfo.mdl}") @@ -119,17 +122,17 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { } } - override fun chunkIterator(v2: Variable, section: SectionPartial?, maxElements : Int?) : Iterator> { + override fun chunkIterator(v2: Variable, wantSection: SectionPartial?, maxElements : Int?) : Iterator> { if (v2.nelems == 0L) { return listOf>().iterator() } - val wantSection = SectionPartial.fill(section, v2.shape) + val section = SectionPartial.fill(wantSection, v2.shape) if (v2.spObject is DataContainerVariable) { val vinfo = v2.spObject if (vinfo.onlyFillValue) { // fill value only, no data val tba = TypedByteArray(v2.datatype, vinfo.fillValue, 0, isBE = vinfo.h5type.isBE) val single = - ArraySection(ArraySingle(wantSection.shape.toIntArray(), v2.datatype, tba.get(0)), wantSection) + ArraySection(ArraySingle(section.shape.toIntArray(), v2.datatype, tba.get(0)), section) return listOf(single).iterator() } } @@ -137,9 +140,9 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { // TODO can we use concurrent reading ?? return if (this.layoutName(v2) == "DataLayoutBTreeVer1") { // H5chunkIterator(header, v2, wantSection) - H5chunkIterator2(this, v2, section) + H5chunkIterator2(this, v2, wantSection) } else { - H5maxIterator(this, v2, wantSection, maxElements ?: 100_000) + H5maxIterator(this, v2, section, maxElements ?: 100_000) } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt index b210ff8a..25c44fbb 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt @@ -554,7 +554,7 @@ internal fun H5builder.readFilterPipelineMessage(state: OpenFileState): FilterPi return FilterPipelineMessage(filters) } -internal class FilterMessage(val filterId: Int, val filterType: FilterType, val name: String, val clientValues: IntArray) +internal data class FilterMessage(val filterId: Int, val filterType: FilterType, val name: String, val clientValues: IntArray) internal data class FilterPipelineMessage(val filters: List) : MessageHeader(MessageType.FilterPipeline) { override fun show() : String { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt index be22fa2f..ac2117cc 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/StructDsl.kt @@ -26,13 +26,11 @@ internal class StructDsl(val name : String, val ba : ByteArray, val isBE: Boolea } // hmmmm if (fld.nelems > 1 && fld.elemSize != 1) { - Array(fld.nelems) { idx -> - when (fld.elemSize) { - 2 -> convertToShort(ba, fld.pos + 2 * idx, isBE) - 4 -> convertToInt(ba, fld.pos + 4 * idx, isBE) - 8 -> convertToLong(ba, fld.pos + 8 * idx, isBE) - else -> getString(fld.fldName) - } + return when (fld.elemSize) { + 2 -> Array(fld.nelems) { idx -> convertToShort(ba, fld.pos + 2 * idx, isBE) } + 4 -> Array(fld.nelems) { idx -> convertToInt(ba, fld.pos + 4 * idx, isBE) } + 8 -> Array(fld.nelems) { idx -> convertToLong(ba, fld.pos + 8 * idx, isBE) } + else -> getString(fld.fldName) } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt index 88ea75c8..31f27944 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/netcdf3/Netcdf3File.kt @@ -29,18 +29,18 @@ internal class Netcdf3File(val filename : String) : Netchdf { override fun type() = header.formatType() override val size : Long get() = raf.size() - override fun readArrayData(v2: Variable, section: SectionPartial?): ArrayTyped { + override fun readArrayData(v2: Variable, wantSection: SectionPartial?): ArrayTyped { if (v2.nelems == 0L) { return ArrayEmpty(v2.shape.toIntArray(), v2.datatype) } - val wantSection : Section = SectionPartial.fill(section, v2.shape) + val section : Section = SectionPartial.fill(wantSection, v2.shape) val vinfo = v2.spObject as VinfoN3 val layout = if (!v2.hasUnlimited()) { - LayoutRegular(vinfo.begin, vinfo.elemSize, wantSection) + LayoutRegular(vinfo.begin, vinfo.elemSize, section) } else { - LayoutRegularSegmented(vinfo.begin, vinfo.elemSize, header.recsize, wantSection) + LayoutRegularSegmented(vinfo.begin, vinfo.elemSize, header.recsize, section) } - return readDataWithLayout(layout, v2, wantSection) + return readDataWithLayout(layout, v2, section) } private fun Variable<*>.hasUnlimited() : Boolean { diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5CompareWithClib.kt similarity index 97% rename from testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt rename to testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5CompareWithClib.kt index 0711129e..94025573 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5CompareWithClib.kt @@ -2,7 +2,6 @@ package com.sunya.netchdf.hdf5 import com.sunya.cdm.api.Datatype import com.sunya.netchdf.* -import com.sunya.netchdf.netcdfClib.NClibFile import com.sunya.netchdf.testfiles.H5Files import com.sunya.netchdf.testfiles.N4Files import com.sunya.netchdf.testutils.testData @@ -93,6 +92,11 @@ class Hdf5Compare { CompareCdmWithClib(testData + "devcdm/hdf5/vlstra.h5") } + @Test + fun chunkedCompoundData() { + compareDataWithClib(testData + "devcdm/hdf5/cuslab.h5", varname = "ArrayOfStructures") + } + //// the following wont work opening as netcdf @Test fun notNetcdf() { diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt index f334d282..c94b714f 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt @@ -22,7 +22,7 @@ class H5readConcurrentTest { @Test fun sanity() { - compareChunkReading(testData + "cdmUnitTest/formats/netcdf4/hiig_forec_20140208.nc", "salt", showStats = true) + compareChunkReading(testData + "cdmUnitTest/formats/netcdf4/hiig_forec_20140208.nc", "salt") } // array reading is failing, btree address == -1 @@ -49,15 +49,16 @@ class H5readConcurrentTest { @Test fun timeH5compareReading() { - val filename = "../core/src/commonTest/data/netcdf4/tiling.nc4" - // val filename = "/home/all/testdata/cdmUnitTest/formats/netcdf4/hiig_forec_20140208.nc" - val varname = "Turbulence_SIGMET_AIRMET" // "salt" + // val filename = "../core/src/commonTest/data/netcdf4/tiling.nc4" + val filename = "/home/all/testdata/cdmUnitTest/formats/netcdf4/hiig_forec_20140208.nc" + // val varname = "Turbulence_SIGMET_AIRMET" // "salt" + val varname = "salt" Hdf5File(filename).use { myfile : Hdf5File -> println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") - println(" ${myvar.fullname()}") + println(" ${myvar.nameAndShape()} nelems = ${myvar.nelems}") val timing = mutableMapOf>() println("readArrayData")