diff --git a/Readme.md b/Readme.md index 5dc44122..d72f00d0 100644 --- a/Readme.md +++ b/Readme.md @@ -1,11 +1,13 @@ # netchdf -_last updated: 6/12/2025_ +_last updated: 7/7/2025_ This is a rewrite in Kotlin of parts of the devcdm and netcdf-java libraries. The intention is to create a maintainable, read-only, pure JVM library allowing full access to netcdf3, netcdf4, hdf4, hdf5, hdf-eos2, and hdf-eos5 data files. +Evaluating if support for superblock 4 is feasible. + Please contact me if you'd like to help out. Especially needed are test datasets from all the important data archives!! ### Building diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 38ad3b94..9f12ecd7 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -8,7 +8,7 @@ version = libs.versions.netchdf.get() kotlin { jvm() -/* + val hostOs = System.getProperty("os.name") val isMingwX64 = hostOs.startsWith("Windows") val arch = System.getProperty("os.arch") @@ -31,8 +31,6 @@ kotlin { else -> throw GradleException("Host OS is not supported.") } - */ - sourceSets { val commonMain by getting { dependencies { @@ -42,11 +40,10 @@ kotlin { implementation(libs.fleeksoft) } } - val jvmMain by - getting { - dependencies { - implementation(libs.slf4j.jvm) - } + val jvmMain by getting { + dependencies { + implementation(libs.slf4j.jvm) + } } val commonTest by getting { dependencies { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2.kt index 31d84012..735ce41d 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2.kt @@ -12,7 +12,7 @@ import com.sunya.netchdf.hdf5.BTree1.Node * TODO ?? Used in readGroupNew( type 5 and 6), readAttributesFromInfoMessage(), FractalHeap, and DHeapId(type 1,2,3,4) */ @OptIn(InternalLibraryApi::class) -internal class BTree2(private val h5: H5builder, owner: String, address: Long, val ndimStorage: Int? = null) : BTreeIF { // BTree2 +internal class BTree2(private val h5: H5builder, owner: String, address: Long, val ndimStorage: Int) : BTreeIF { // BTree2 val btreeType: Int private val nodeSize: Int // size in bytes of btree nodes private val recordSize: Short// size in bytes of btree records @@ -226,8 +226,8 @@ internal class BTree2(private val h5: H5builder, owner: String, address: Long, v 7 -> Record70(state) // TODO wrong 8 -> Record8(state) 9 -> Record9(state) - 10 -> Record10(state, ndimStorage!!) // TODO wrong, whats ndims? - 11 -> Record11(state, ndimStorage!!) // TODO wrong, whats ndims? + 10 -> Record10(state, ndimStorage - 1) // TODO wrong, whats ndims? + 11 -> Record11(state, ndimStorage - 1) // TODO wrong, whats ndims? else -> throw IllegalStateException() } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt index 2ad421e9..40884a0b 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FractalHeap.kt @@ -206,7 +206,7 @@ internal class FractalHeap(private val h5: H5builder, forWho: String, address: L when (subtype) { 1, 2 -> { val btree = if (btreeHugeObjects == null) { - val local = BTree2(h5, "FractalHeap btreeHugeObjects", btreeAddressHugeObjects) + val local = BTree2(h5, "FractalHeap btreeHugeObjects", btreeAddressHugeObjects, 0) require(local.btreeType == subtype) local.readEntries() } else btreeHugeObjects diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt index ec0b0997..d5113e10 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt @@ -76,6 +76,66 @@ internal class H5chunkReader(val h5 : H5builder) { */ } + internal fun readSingleChunk(v2: Variable, wantSection: Section): ArrayTyped { + val vinfo = v2.spObject as DataContainerVariable + val h5type = vinfo.h5type + + val elemSize = vinfo.storageDims[vinfo.storageDims.size - 1].toInt() // last one is always the elements size + val datatype = vinfo.h5type.datatype() + + val wantSpace = IndexSpace(wantSection) + val sizeBytes = wantSpace.totalElements * elemSize + if (sizeBytes <= 0 || sizeBytes >= Int.MAX_VALUE) { + throw RuntimeException("Illegal nbytes to read = $sizeBytes") + } + val mdl = vinfo.mdl as DataLayoutSingleChunk4 + if (mdl.isFiltered) throw UnsupportedOperationException("readSingleChunk doesnt support filtered data") + val ba = ByteArray(sizeBytes.toInt()) + + val state = OpenFileState(mdl.heapAddress, vinfo.h5type.isBE) + h5.raf.readByteArray(state, ba.size) + + return h5.processDataIntoArray(ba, vinfo.h5type.isBE, datatype, v2.shape.toIntArray(), h5type, elemSize) as ArrayTyped + + /* + val varShape = v2.shape + val chunk = IntArray(mdl.dims.size - 1) { mdl.dims[it] } // remove the element size + val nchunkElems = chunk.computeSize() + val odo = IndexND(IndexSpace(chunk), varShape) + + var countChunkElems = 0L + //var transferBytes = 0 + val state = OpenFileState(0L, vinfo.h5type.isBE) + for (dataChunk: FilteredChunk in index!!.filteredChunks) { + // TODO we need to know the dataChunk's IndexSpace, ie its position with varShape. + if (debugChunking) println("$countChunkElems == ${odo.current.contentToString()}}") + + val dataSection = IndexSpace(v2.rank, odo.current, vinfo.storageDims) + val chunker = Chunker(dataSection, wantSpace) + /* if (dataChunk.isMissing()) { + if (debugChunking) println(" missing ${dataChunk.show(tiledData.tiling)}") + chunker.transferMissing(vinfo.fillValue, elemSize, ba) + } else { */ + //if (debugChunking) println(" chunk=${dataChunk.show(tiledData.tiling)}") + state.pos = dataChunk.address + val chunkData = h5.raf.readByteArray(state, dataChunk.chunkSize) + val filteredData = filters.apply(chunkData, dataChunk.filterMask) + chunker.transferBA(filteredData, 0, elemSize, ba, 0) // this iterates overs Chunker's chunks + //transferChunks += chunker.transferChunks + // } + countChunkElems += nchunkElems + odo.set(countChunkElems) + } + + */ + + // val filteredData = if (dataChunk.filterMask() == null) chunkData + // else filters.apply(chunkData, dataChunk.filterMask()!!) + + + + } + /* internal fun readBtreeVer1(v2: Variable, wantSection: Section): ArrayTyped { val vinfo = v2.spObject as DataContainerVariable @@ -170,7 +230,6 @@ internal class H5chunkReader(val h5 : H5builder) { val shape = wantSpace.shape.toIntArray() - return if (h5type.datatype5 == Datatype5.Vlen) { h5.processVlenIntoArray(h5type, shape, ba, wantSpace.totalElements.toInt(), elemSize) } else { diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt index 16e72f33..ab6274a7 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt @@ -59,7 +59,7 @@ internal fun H5builder.readGroupNew( check(btreeAddress >= 0) { "no valid btree for GroupNew with Fractal Heap" } // read in btree and all entries - val btree = BTree2(this, parent.name, btreeAddress) + val btree = BTree2(this, parent.name, btreeAddress, 0) for (e in btree.readEntries()) { var heapId: ByteArray = when (btree.btreeType) { 5 -> (e.record as BTree2.Record5).heapId diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt index fe67f0e5..69a0be54 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt @@ -50,12 +50,12 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { alldata.section(wantSection) } else if (vinfo.mdl.isContiguous) { header.readRegularData(vinfo, v2.datatype, wantSection) - // } else if (vinfo.mdl is DataLayoutBTreeVer1) { - // H5chunkReader(header).readBtreeVer1(v2, wantSection) - } else if (vinfo.mdl is DataLayoutFixedArray4) { - H5chunkReader(header).readFixedArray(v2, wantSection) } else if (vinfo.mdl is DataLayoutBTreeVer1 || vinfo.mdl is DataLayoutBtreeVer2) { H5chunkReader(header).readBtreeVer12(v2, wantSection) + } else if (vinfo.mdl is DataLayoutSingleChunk4) { + H5chunkReader(header).readSingleChunk(v2, wantSection) + } else if (vinfo.mdl is DataLayoutFixedArray4) { + H5chunkReader(header).readFixedArray(v2, wantSection) } else { throw RuntimeException("Unsupported data layer type ${vinfo.mdl}") } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt index 2a11be30..4564c0c7 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageDataLayout.kt @@ -2,6 +2,7 @@ package com.sunya.netchdf.hdf5 +import com.sunya.cdm.api.computeSize import com.sunya.cdm.iosp.OpenFileState import com.sunya.cdm.util.InternalLibraryApi @@ -129,6 +130,10 @@ internal fun H5builder.readDataLayoutMessage(state : OpenFileState) : DataLayout // version 4, layoutClass = 2 is too complex for structdls if (layoutClass == 2) { + val nextBytes = raf.readByteArray(state.copy(), 40) + println("version 4, layoutClass = 2 ${this.raf.location()}") + println(" nextBytes after chunkIndexingType ${nextBytes.contentToString()}") + // this structure is too complex for structdls val version = raf.readByte(state) val layoutClass = raf.readByte(state) @@ -136,27 +141,66 @@ internal fun H5builder.readDataLayoutMessage(state : OpenFileState) : DataLayout val rank = raf.readByte(state).toInt() val dimSizeLength = raf.readByte(state) val dims = IntArray(rank) { this.readVariableSizeDimension(state, dimSizeLength) } // TODO is dimSizeLength correct ?? + var chunkSize = dims.computeSize() + val chunkIndexingType = raf.readByte(state).toInt() return when (chunkIndexingType) { 1 -> { // VII.A single chunk index - val chunkSize = this.readLength(state) + // #define H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS 0x01 // no filter + // #define H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER 0x02 // has a filter + + // udata->chunk_block.offset = idx_info->storage->idx_addr; + // if (idx_info->layout->flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + // udata->chunk_block.length = idx_info->storage->u.single.nbytes; + // udata->filter_mask = idx_info->storage->u.single.filter_mask; + // } /* end if */ + // else { + // udata->chunk_block.length = idx_info->layout->size; + // udata->filter_mask = 0; + // } /* end else */ + + // "The following information exists only when the chunk is filtered. + // In other words, when H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER (bit 1) is enabled in the field flags." + // TODO not clear val nextBytes = raf.readByteArray(state.copy(), 40) println("SingleChunk ${this.raf.location()}") - println(" chunkSize $chunkSize nextBytes ${nextBytes.contentToString()}") - // https://github.com/HDFGroup/hdf5/issues/5610 - // The second field should be "Filter mask" for the chunk, which indicates the filter to skip for the dataset chunk. - // Each filter has an index number in the pipeline; if that filter is skipped, the bit corresponding to its index is set. - val filterMask = raf.readInt(state) - /* repeat ( 32) { idx -> - val isSet = isBitSet(filterMask, idx) - println(" idx = $idx isSet = $isSet") - } */ + println(" nextBytes after chunkIndexingType ${nextBytes.contentToString()}") + + var filterMask : Int? = null + if (isBitSet(flags.toInt(), 1)) { + // Indexing Type Information (variable size) + chunkSize = this.readLength(state).toInt() + + // https://github.com/HDFGroup/hdf5/issues/5610 + // The second field should be "Filter mask" for the chunk, which indicates the filter to skip for the dataset chunk. + // Each filter has an index number in the pipeline; if that filter is skipped, the bit corresponding to its index is set. + // It would be surprising to have a Filter Mask here, since that usually references a FilterPipeline message, but there is none. + + // /home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__testfilter_reg.nc + // sizeOfFilteredChunk 1024 nextBytes [0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 21, 28, 0, 4, 0, 0, 0, 3, 2, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1] + + // filtered, flags == 2; /home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/examples__bzip2.nc + // sizeOfFilteredChunk 501 nextBytes [0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 21, 28, 0, 4, 0, 0, 0, 3, 2, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1] + + // TODO not clear, number and meaning of the fields. Possibly embedding the filter pipeline here, instead of a seperate message ?? + // uint32_t filter_mask; /* Excluded filters for chunk */ seems theres only 32 filters? must be a seperate mechansism for extensions + filterMask = raf.readInt(state) // "This field contains filters for the chunk." + repeat ( 32) { idx -> + val isSet = isBitSet(filterMask, idx) + if (isSet) println(" idx = $idx isSet = $isSet") + } + println(" sizeOfFilteredChunk $chunkSize filterMask $filterMask") + } + + // [0, 4, 0, 0, 0, 0, 0, 0, + // 0, 0, 0, 0, 3, 27, 0, 0, + // 0, 0, 0, 0, 21, 28, 0, 4, + // 0, 0, 0, 3, 2, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1] + // Address of the single chunk. size specified in “Size of Lengths” field in the superblock. // The address may be undefined if the chunk or index storage is not allocated yet. val chunkAddress = this.readLength(state) - println(" filterMask $filterMask chunkAddress $chunkAddress") - - DataLayoutSingleChunk4(flags, dims, chunkSize, chunkAddress) + DataLayoutSingleChunk4(flags, dims, chunkSize = chunkSize, chunkAddress, filterMask) } 2 -> { // VII.B implicit index // Address of the array of dataset chunks. @@ -180,7 +224,9 @@ internal fun H5builder.readDataLayoutMessage(state : OpenFileState) : DataLayout val minElements = raf.readByte(state) val pageBits = raf.readByte(state) val indexAddress = raf.readLong(state) // probably wrong - DataLayoutExtensibleArray4(flags, dims, maxBits, indexElements, minPointers, minElements, pageBits, indexAddress) + val result = DataLayoutExtensibleArray4(flags, dims, maxBits, indexElements, minPointers, minElements, pageBits, indexAddress) + println(result.show()) + result } 5 -> { // VII.E version 2 B-tree index val nodeSize = raf.readInt(state) @@ -192,7 +238,7 @@ internal fun H5builder.readDataLayoutMessage(state : OpenFileState) : DataLayout } else -> throw RuntimeException() } - val address = raf.readLong(state) // TODO read address ?? + // val address = raf.readLong(state) // TODO read address ?? } } throw RuntimeException() @@ -245,8 +291,9 @@ internal data class DataLayoutContiguous3(val dataAddress: Long, val dataSize: L } // 4 -internal data class DataLayoutSingleChunk4(val flags: Byte, val dims: IntArray, val chunkSize: Long, val heapAddress: Long) : DataLayoutMessage() { - override fun show(): String = "${super.show()} flags=$flags dims=$dims chunkSize=$chunkSize heapAddress=$heapAddress" +internal data class DataLayoutSingleChunk4(val flags: Byte, val dims: IntArray, val chunkSize: Int, val heapAddress: Long, val filterMask: Int?) : DataLayoutMessage() { + val isFiltered = isBitSet(flags.toInt(), 1) + override fun show(): String = "${super.show()} flags=$flags dims=$dims heapAddress=$heapAddress chunkSize=$chunkSize" } internal data class DataLayoutImplicit4(val flags: Byte, val dims: IntArray, val address: Long) : DataLayoutMessage() { override fun show(): String = "${super.show()} flags=$flags dims=$dims address=$address" @@ -256,7 +303,7 @@ internal data class DataLayoutFixedArray4(val flags: Byte, val dims: IntArray, v } internal data class DataLayoutExtensibleArray4(val flags: Byte, val dims: IntArray, val maxBits: Byte, val indexElements: Byte, val minPointers: Byte, val minElements: Byte, val pageBits: Byte, val indexAddress: Long) : DataLayoutMessage() { - override fun show(): String = "${super.show()} flags=$flags dims=$dims maxBits=$maxBits indexElements=$indexElements " + + override fun show(): String = "${super.show()} flags=$flags dims=${dims.contentToString()} maxBits=$maxBits indexElements=$indexElements " + "minPointers=$minPointers minElements=$minPointers pageBits=$pageBits indexAddress=$indexAddress" } internal data class DataLayoutBtreeVer2(val flags: Byte, val dims: IntArray, val nodeSize: Int, val splitPercent: Byte, val mergePercent: Byte, val heapAddress: Long) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt index 81c70037..a5b1bcfb 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/MessageHeader.kt @@ -867,7 +867,7 @@ private fun H5builder.readAttributesFromInfoMessage( val btreeAddress: Long = attributeOrderBtreeAddress ?: attributeNameBtreeAddress if (btreeAddress < 0 || fractalHeapAddress < 0) return emptyList() - val btree2 = BTree2(this, "AttributeInfoMessage", btreeAddress) + val btree2 = BTree2(this, "AttributeInfoMessage", btreeAddress, 0) val fractalHeap = FractalHeap(this, "AttributeInfoMessage", fractalHeapAddress) val list = mutableListOf() diff --git a/core/src/commonMain/kotlin/com/sunya/netchdfc/NetchdfCApi.kt b/core/src/commonMain/kotlin/com/sunya/netchdfc/NetchdfCApi.kt index 47f7d0c4..d4302c68 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdfc/NetchdfCApi.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdfc/NetchdfCApi.kt @@ -3,6 +3,28 @@ package com.sunya.netchdfc import com.sunya.cdm.api.* import com.sunya.cdm.array.ArrayInt +/* +import kotlinx.cinterop.CPointer +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.IntVar +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.pin +import kotlin.experimental.ExperimentalNativeApi +import kotlin.native.CName + +@OptIn(ExperimentalNativeApi::class, ExperimentalForeignApi::class) +@CName("getPinnedIntArrayPointer") +fun getPinnedIntArrayPointer(array: IntArray): CPointer { + return array.pin().addressOf(0) +} + +// for testing +@OptIn(ExperimentalForeignApi::class) +fun testCArray(): CPointer { + return getPinnedIntArrayPointer(intArrayOf(1,2,3,4,5)) +} +*/ + fun version() : String { return "netchdf version 0.4.0" } @@ -27,4 +49,4 @@ class ArrayIntSection(val varName: String, val varShape: LongArray, val rank: In internal fun Section.toSectionPartial() : SectionPartial { return SectionPartial(this.ranges) -} \ No newline at end of file +} diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/H5superblockTest.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/H5superblockTest.kt index 6f7311fa..e3ade267 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/H5superblockTest.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/H5superblockTest.kt @@ -2,6 +2,7 @@ package com.sunya.netchdf.hdf5 import com.sunya.netchdf.testfiles.SuperblockFiles import com.sunya.netchdf.testfiles.testData +import com.sunya.netchdf.testutil.readNetchdfData import kotlin.test.* @@ -175,14 +176,14 @@ class H5superblockTest { } // /home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__testfilter_reg.nc - // chunkSize 1024 next20bytes [0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 21, 28, 0, 4, 0, 0, 0, 3] - // im guessing the 0 means no filter. 4^5 = 1024 + // chunkSize 1024 nextBytes [0, 0, 0, 0, 3, 27, 0, 0, 0, 0, 0, 0, 21, 28, 0, 4, 0, 0, 0, 3, 2, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1] + // filterMask 0 chunkAddress 6915 + // im guessing the 0 means no filter. 4^5 = 1024 // var = class com.sunya.cdm.array.ArrayFloat shape=[4, 4, 4, 4] data=0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0,51.0,52.0,53.0,54.0,55.0,56.0,57.0,58.0,59.0,60.0,61.0,62.0,63.0,64.0,65.0,66.0,67.0,68.0,69.0,70.0,71.0,72.0,73.0,74.0,75.0,76.0,77.0,78.0,79.0,80.0,81.0,82.0,83.0,84.0,85.0,86.0,87.0,88.0,89.0,90.0,91.0,92.0,93.0,94.0,95.0,96.0,97.0,98.0,99.0,100.0,101.0,102.0,103.0,104.0,105.0,106.0,107.0,108.0,109.0,110.0,111.0,112.0,113.0,114.0,115.0,116.0,117.0,118.0,119.0,120.0,121.0,122.0,123.0,124.0,125.0,126.0,127.0,128.0,129.0,130.0,131.0,132.0,133.0,134.0,135.0,136.0,137.0,138.0,139.0,140.0,141.0,142.0,143.0,144.0,145.0,146.0,147.0,148.0,149.0,150.0,151.0,152.0,153.0,154.0,155.0,156.0,157.0,158.0,159.0,160.0,161.0,162.0,163.0,164.0,165.0,166.0,167.0,168.0,169.0,170.0,171.0,172.0,173.0,174.0,175.0,176.0,177.0,178.0,179.0,180.0,181.0,182.0,183.0,184.0,185.0,186.0,187.0,188.0,189.0,190.0,191.0,192.0,193.0,194.0,195.0,196.0,197.0,198.0,199.0,200.0,201.0,202.0,203.0,204.0,205.0,206.0,207.0,208.0,209.0,210.0,211.0,212.0,213.0,214.0,215.0,216.0,217.0,218.0,219.0,220.0,221.0,222.0,223.0,224.0,225.0,226.0,227.0,228.0,229.0,230.0,231.0,232.0,233.0,234.0,235.0,236.0,237.0,238.0,239.0,240.0,241.0,242.0,243.0,244.0,245.0,246.0,247.0,248.0,249.0,250.0,251.0,252.0,253.0,254.0,255.0 @Test - fun testSingleChunkNoFilter() { - val filename = "/home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__testfilter_reg.nc" - val ok = openH5(filename, "var", showCdl = true) - assertTrue(ok) + fun testSingleChunkBZipFilter() { + readNetchdfData(testData + "netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__testfilter_reg.nc", + "var", showCdl = true, showData = true) } // h5dump -H /home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__tst_chunks.nc @@ -358,8 +359,109 @@ class H5superblockTest { assertTrue(ok) } + // /home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10:$ h5dump -H /home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__tst_dims3.nc + //HDF5 "/home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__tst_dims3.nc" { + //GROUP "/" { + // ATTRIBUTE "_NCProperties" { + // DATATYPE H5T_STRING { + // STRSIZE 35; + // STRPAD H5T_STR_NULLTERM; + // CSET H5T_CSET_ASCII; + // CTYPE H5T_C_S1; + // } + // DATASPACE SCALAR + // } + // DATASET "var" { + // DATATYPE H5T_STD_I32LE + // DATASPACE SIMPLE { ( 1, 3 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + // ATTRIBUTE "DIMENSION_LIST" { + // DATATYPE H5T_VLEN { H5T_REFERENCE { H5T_STD_REF_OBJECT }} + // DATASPACE SIMPLE { ( 2 ) / ( 2 ) } + // } + // ATTRIBUTE "_Netcdf4Coordinates" { + // DATATYPE H5T_STD_I32LE + // DATASPACE SIMPLE { ( 2 ) / ( 2 ) } + // } + // } + // DATASET "x" { + // DATATYPE H5T_IEEE_F32BE + // DATASPACE SIMPLE { ( 0 ) / ( H5S_UNLIMITED ) } + // ATTRIBUTE "CLASS" { + // DATATYPE H5T_STRING { + // STRSIZE 16; + // STRPAD H5T_STR_NULLTERM; + // CSET H5T_CSET_ASCII; + // CTYPE H5T_C_S1; + // } + // DATASPACE SCALAR + // } + // ATTRIBUTE "NAME" { + // DATATYPE H5T_STRING { + // STRSIZE 64; + // STRPAD H5T_STR_NULLTERM; + // CSET H5T_CSET_ASCII; + // CTYPE H5T_C_S1; + // } + // DATASPACE SCALAR + // } + // ATTRIBUTE "REFERENCE_LIST" { + // DATATYPE H5T_COMPOUND { + // H5T_REFERENCE { H5T_STD_REF_OBJECT } "dataset"; + // H5T_STD_I32LE "dimension"; + // } + // DATASPACE SIMPLE { ( 1 ) / ( 1 ) } + // } + // ATTRIBUTE "_Netcdf4Dimid" { + // DATATYPE H5T_STD_I32LE + // DATASPACE SCALAR + // } + // } + // DATASET "y" { + // DATATYPE H5T_STD_I32LE + // DATASPACE SIMPLE { ( 3 ) / ( H5S_UNLIMITED ) } + // ATTRIBUTE "CLASS" { + // DATATYPE H5T_STRING { + // STRSIZE 16; + // STRPAD H5T_STR_NULLTERM; + // CSET H5T_CSET_ASCII; + // CTYPE H5T_C_S1; + // } + // DATASPACE SCALAR + // } + // ATTRIBUTE "NAME" { + // DATATYPE H5T_STRING { + // STRSIZE 2; + // STRPAD H5T_STR_NULLTERM; + // CSET H5T_CSET_ASCII; + // CTYPE H5T_C_S1; + // } + // DATASPACE SCALAR + // } + // ATTRIBUTE "REFERENCE_LIST" { + // DATATYPE H5T_COMPOUND { + // H5T_REFERENCE { H5T_STD_REF_OBJECT } "dataset"; + // H5T_STD_I32LE "dimension"; + // } + // DATASPACE SIMPLE { ( 1 ) / ( 1 ) } + // } + // ATTRIBUTE "_Netcdf4Dimid" { + // DATATYPE H5T_STD_I32LE + // DATASPACE SCALAR + // } + // } + //} + //} + // but do we trust ncdump with superblock 4? + // netcdf nc_test4__tst_dims3 { + // dimensions: + // y = UNLIMITED ; // (3 currently) + // x = UNLIMITED ; // (3 currently) + // variables: + // int y(y) ; + // int var(y, x) ; + // } @Test - fun testBTreeVer2() { + fun testExtensibleArrayIndex() { val filename = "/home/all/testdata/netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_10/nc_test4__tst_dims3.nc" val ok = openH5(filename, "var", showCdl = true) assertTrue(ok) diff --git a/core/src/nativeMain/kotlin/Main.kt b/core/src/nativeMain/kotlin/Main.kt new file mode 100644 index 00000000..32a31ed2 --- /dev/null +++ b/core/src/nativeMain/kotlin/Main.kt @@ -0,0 +1,5 @@ + +fun main(args: Array) { + // Read the input value. + println("Hello, enter your name:") +} \ No newline at end of file