diff --git a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/BitShuffleFilter.kt b/cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/BitShuffleFilter.kt similarity index 97% rename from cli/src/main/kotlin/com/sunya/netchdf/hdf5/BitShuffleFilter.kt rename to cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/BitShuffleFilter.kt index 3141a751..4d9e251d 100644 --- a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/BitShuffleFilter.kt +++ b/cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/BitShuffleFilter.kt @@ -1,5 +1,8 @@ -package com.sunya.netchdf.hdf5 +package com.sunya.netchdf.hdf5filters +import com.sunya.netchdf.hdf5.H5filterIF +import com.sunya.netchdf.hdf5.makeIntFromBEBytes +import com.sunya.netchdf.hdf5.makeLongFromBEBytes import net.jpountz.lz4.LZ4Factory // seems to handle LZ4_COMPRESSION combined with bit shuffle diff --git a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/Lz4Filter.kt b/cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/Lz4Filter.kt similarity index 90% rename from cli/src/main/kotlin/com/sunya/netchdf/hdf5/Lz4Filter.kt rename to cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/Lz4Filter.kt index 3878936c..6b6b98eb 100644 --- a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/Lz4Filter.kt +++ b/cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/Lz4Filter.kt @@ -1,5 +1,8 @@ -package com.sunya.netchdf.hdf5 +package com.sunya.netchdf.hdf5filters +import com.sunya.netchdf.hdf5.H5filterIF +import com.sunya.netchdf.hdf5.makeIntFromBEBytes +import com.sunya.netchdf.hdf5.makeLongFromBEBytes import net.jpountz.lz4.LZ4Factory import kotlin.math.min diff --git a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/LzfFilter.kt b/cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/LzfFilter.kt similarity index 90% rename from cli/src/main/kotlin/com/sunya/netchdf/hdf5/LzfFilter.kt rename to cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/LzfFilter.kt index 278090fe..e42c3747 100644 --- a/cli/src/main/kotlin/com/sunya/netchdf/hdf5/LzfFilter.kt +++ b/cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/LzfFilter.kt @@ -1,7 +1,8 @@ -package com.sunya.netchdf.hdf5 +package com.sunya.netchdf.hdf5filters import com.ning.compress.lzf.LZFException import com.ning.compress.lzf.util.ChunkDecoderFactory +import com.sunya.netchdf.hdf5.H5filterIF class LzfFilter() : H5filterIF { override fun id() = 32000 diff --git a/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt b/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt index 5fc210dd..ffd0771f 100644 --- a/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt +++ b/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt @@ -1,6 +1,9 @@ package com.sunya.netchdf.hdf5 import com.sunya.cdm.api.computeSize +import com.sunya.netchdf.hdf5filters.BitShuffleFilter +import com.sunya.netchdf.hdf5filters.Lz4Filter +import com.sunya.netchdf.hdf5filters.LzfFilter import com.sunya.netchdf.openNetchdfFile import kotlin.test.Test diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt b/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt index 8393182b..d94f0476 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt @@ -13,7 +13,7 @@ data class Variable( ) { val name = makeValidCdmObjectName(orgName) val rank : Int = dimensions.size - val shape : LongArray = dimensions.map { it.length }.toLongArray() // why LongArray ?? + val shape : LongArray = dimensions.map { it.length }.toLongArray() // if you allow dimensions to be long, you need LongArray shape. val nelems : Long = this.shape.computeSize() fun fullname() : String { diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/AtomicDouble.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/AtomicDouble.kt deleted file mode 100644 index 526bf49d..00000000 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/AtomicDouble.kt +++ /dev/null @@ -1,47 +0,0 @@ -package com.sunya.netchdf.testutil - -import kotlin.concurrent.atomics.AtomicReference // TODO kmm not quite ready? Not sure I have semantics right. -import kotlin.concurrent.atomics.ExperimentalAtomicApi - -@OptIn(ExperimentalAtomicApi::class) -class AtomicDouble(initialValue: Double) { - private val atomicReference = AtomicReference(initialValue) - - fun get(): Double = atomicReference.load() - - fun set(newValue: Double) { - atomicReference.store(newValue) - } - - fun getAndAdd(delta: Double): Double { - while (true) { - val current = atomicReference.load() - val new = current + delta - if (atomicReference.compareAndSet(current, new)) { - return current - } - } - } -} - -// not quite ready -/* -import kotlinx.atomicfu.* - -class AtomicDouble(initialValue: Double) { - private val _value: AtomicRef = atomic(initialValue) - - fun get(): Double = _value.value - - fun set(newValue: Double) { - _value.value = newValue - } - - fun getAndSet(newValue: Double): Double = _value.getAndSet(newValue) - - fun compareAndSet(expect: Double, update: Double): Boolean = _value.compareAndSet(expect, update) - - // fun addAndGet(delta: Double): Double = _value.getAndAdd(delta) + delta - - fun getAndAdd(delta: Double): Double = _value.getAndAdd(delta) -} */ \ No newline at end of file diff --git a/testclibs/build.gradle.kts b/testclibs/build.gradle.kts index fd645f57..52c223ac 100644 --- a/testclibs/build.gradle.kts +++ b/testclibs/build.gradle.kts @@ -5,8 +5,6 @@ plugins { dependencies { implementation(project(":core")) implementation(project(":testfiles")) - // implementation(project(":cli")) - //implementation(files("/home/stormy/dev/github/netcdf/netchdf/cli/build/libs/cli.jar")) implementation(libs.okio) implementation(libs.fleeksoft) diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt index 03184759..f590a828 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt @@ -7,7 +7,6 @@ import com.sunya.cdm.util.InternalLibraryApi import com.sunya.netchdf.* import com.sunya.netchdf.hdf4Clib.Hdf4ClibFile import com.sunya.netchdf.testfiles.H4Files -import com.sunya.netchdf.testutils.Stats import com.sunya.netchdf.testutils.testData import kotlin.test.* @@ -18,13 +17,6 @@ class H4Ccompare { return H4Files.files() } - fun afterAll() { - if (versions.size > 0) { - versions.keys.forEach { println("$it = ${versions[it]!!.size} files") } - } - Stats.show() - } - private val versions = mutableMapOf>() } @@ -135,12 +127,6 @@ class H4Ccompare { compareData(testData + "devcdm/hdf4/TOVS_BROWSE_MONTHLY_AM_B861001.E861031_NF.HDF", "Raster_Image_#0") } - @Test - fun problemReadData() { - val filename = testData + "/devcdm/hdfeos2/MISR_AM1_GP_GMP_P040_O003734_05.eos" - readNetchdfData(filename, null, null, true) - } - ////////////////////////////////////////////////////////////////////// @Test @@ -249,11 +235,4 @@ class H4Ccompare { compareSelectedDataWithClib(filename) { it.datatype == Datatype.CHAR } // || it.datatype == Datatype.STRING } } } - - //@Test - fun testIterateWithClib() { - files().forEach { filename -> - compareIterateWithClib(filename) - } - } } \ No newline at end of file diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt index 45c704de..0711129e 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt @@ -15,11 +15,11 @@ import kotlin.test.assertTrue class Hdf5Compare { companion object { - @JvmStatic fun files(): Iterator { - // 10 of 114 fail, because we compare with netcdf4 instead of hdf5 c library - - return H5Files.files() + return sequenceOf( + N4Files.files().asSequence(), + H5Files.files().asSequence(), + ).flatten().iterator() } } @@ -51,6 +51,67 @@ class Hdf5Compare { compareDataWithClib(filename) } + // a compound with a member thats a type thats not a seperate typedef. + // the obvious thing to do is to be able to add a typedef when processing the member. + // or look for it when building H5group + @Test + fun compoundEnumTypedef() { + CompareCdmWithClib(testData + "devcdm/hdf5/enumcmpnd.h5") + } + + @Test + fun vlenData() { + CompareCdmWithClib(testData + "devcdm/netcdf4/tst_vlen_data.nc4") + compareDataWithClib(testData + "devcdm/netcdf4/tst_vlen_data.nc4") + } + + @Test + fun compoundData() { + CompareCdmWithClib(testData + "devcdm/netcdf4/tst_compounds.nc4") + compareDataWithClib(testData + "devcdm/netcdf4/tst_compounds.nc4") + } + + @Test + fun stringData() { + CompareCdmWithClib(testData + "devcdm/netcdf4/tst_strings.nc") + compareDataWithClib(testData + "devcdm/netcdf4/tst_strings.nc") + } + + @Test + fun opaqueAttribute() { + CompareCdmWithClib(testData + "devcdm/netcdf4/tst_opaque_data.nc4") + } + + @Test + fun testIterateDataSumInfinite() { + CompareCdmWithClib(testData + "cdmUnitTest/formats/hdf5/StringsWFilter.h5") + compareDataWithClib(testData + "cdmUnitTest/formats/hdf5/StringsWFilter.h5", varname = "/observation/matrix/data") + } + + @Test + fun vlstra() { + CompareCdmWithClib(testData + "devcdm/hdf5/vlstra.h5") + } + + //// the following wont work opening as netcdf + @Test + fun notNetcdf() { + CompareCdmWithClib(testData + "devcdm/hdf5/compound_complex.h5", showCdl = true) + CompareCdmWithClib(testData + "devcdm/hdf5/bitfield.h5", showCdl = true) + CompareCdmWithClib(testData + "devcdm/hdf5/SDS_array_type.h5", showCdl = true) + } + + @Test + fun privateTypedef() { + CompareCdmWithClib(testData + "devcdm/hdf5/bitop.h5", showCdl = true) + } + + @Test + fun problemCompareCdl() { + CompareCdmWithClib(testData + "devcdm/netcdf4/testNestedStructure.nc", showCdl = true) + } + + //////////////////////////////////////////////////////////////////// @Test diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5openTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5openTest.kt deleted file mode 100644 index 6f7e54f2..00000000 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5openTest.kt +++ /dev/null @@ -1,242 +0,0 @@ -package com.sunya.netchdf.hdf5 - -import com.sunya.cdm.api.Netchdf -import com.sunya.cdm.api.Variable -import com.sunya.cdm.api.chunkConcurrent -import com.sunya.cdm.array.ArrayTyped -import com.sunya.netchdf.* -import com.sunya.netchdf.testfiles.H5Files -import com.sunya.netchdf.testfiles.N4Files -import com.sunya.netchdf.testutils.AtomicDouble -import com.sunya.netchdf.testutils.Stats -import com.sunya.netchdf.testutils.testData - -import kotlin.test.* -import kotlin.system.measureNanoTime - -// Sanity check read Hdf5File header, for non-netcdf4 files -class Hdf5openTest { - - companion object { - fun files(): Iterator { - return sequenceOf( - N4Files.files().asSequence(), - H5Files.files().asSequence(), - ).flatten().iterator() - } - - fun beforeAll() { - Stats.clear() - } - - fun afterAll() { - Stats.show() - } - } - - @Test - fun hasLinkName() { - openH5(testData + "cdmUnitTest/formats/hdf5/aura/MLS-Aura_L2GP-BrO_v01-52-c01_2007d029.he5") - } - - @Test - fun problemWrf() { - readH5concurrent(testData + "cdmUnitTest/formats/hdf5/wrf/wrf_out_par.h5") - } - - // a compound with a member thats a type thats not a seperate typedef. - // the obvious thing to do is to be able to add a typedef when processing the member. - // or look for it when building H5group - @Test - fun compoundEnumTypedef() { - openH5(testData + "devcdm/hdf5/enumcmpnd.h5") - } - - @Test - fun vlenData() { - readNetchdfData(testData + "devcdm/netcdf4/tst_vlen_data.nc4") - compareDataWithClib(testData + "devcdm/netcdf4/tst_vlen_data.nc4") - } - - @Test - fun compoundData() { - readNetchdfData(testData + "devcdm/netcdf4/tst_compounds.nc4") - compareDataWithClib(testData + "devcdm/netcdf4/tst_compounds.nc4") - } - - @Test - fun stringData() { - readNetchdfData(testData + "devcdm/netcdf4/tst_strings.nc") - compareDataWithClib(testData + "devcdm/netcdf4/tst_strings.nc") - } - - @Test - fun opaqueAttribute() { - openH5(testData + "devcdm/netcdf4/tst_opaque_data.nc4") - } - - @Test - fun testIterateDataSumInfinite() { - // readH5(testData + "devcdm/hdf5/zip.h5", "/Data/Compressed_Data") - compareDataWithClib(testData + "cdmUnitTest/formats/hdf5/StringsWFilter.h5", "/observation/matrix/data") - readH5concurrent(testData + "cdmUnitTest/formats/hdf5/StringsWFilter.h5", "/observation/matrix/data") - } - - @Test - fun vlstra() { - CompareCdmWithClib(testData + "devcdm/hdf5/vlstra.h5") - } - - //// the following wont work opening as netcdf - @Test - fun notNetcdf() { - CompareCdmWithClib(testData + "devcdm/hdf5/compound_complex.h5", showCdl = true) - CompareCdmWithClib(testData + "devcdm/hdf5/bitfield.h5", showCdl = true) - CompareCdmWithClib(testData + "devcdm/hdf5/SDS_array_type.h5", showCdl = true) - } - - @Test - fun privateTypedef() { - CompareCdmWithClib(testData + "devcdm/hdf5/bitop.h5", showCdl = true) - } - - @Test - fun problemCompareCdl() { - CompareCdmWithClib(testData + "devcdm/netcdf4/testNestedStructure.nc", showCdl = true) - } - - /////////////////////////////////////////////////////////////////////////////////// - - @Test - fun testOpenH5all() { - files().forEach { filename -> - openH5(filename, null) - } - } - - @Test - fun testShowNcHeader() { - files().forEach { filename -> - if (filename.endsWith("compound_complex.h5") || - filename.endsWith("bitop.h5") || - filename.endsWith("bitfield.h5") || - filename.endsWith("SDS_array_type.h5") - ) return - - showNcHeader(filename) - } - } - - @Test - fun testCdlWithClibAll() { - files().forEach { filename -> - CompareCdmWithClib(filename) - } - } - - @Test - fun testReadNetchdfData() { - files().forEach { filename -> - readNetchdfData(filename) - } - } - - @Test - fun testReadConcurrent() { - files().forEach { filename -> - readH5concurrent(filename, null) - } - } - - ///////////////////////////////////////////////////////// - - fun openH5(filename: String, varname : String? = null) { - println("=================") - println(filename) - openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF5).use { h5file -> - println(h5file!!.cdl()) - h5file.rootGroup().allVariables().forEach { println(" ${it.fullname()}") } - - if (varname != null) { - val h5var = h5file.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") - val h5data = h5file.readArrayData(h5var) - println(" $varname = $h5data") - } - } - } - - fun readH5concurrent(filename: String, varname : String? = null) { - openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF5).use { myfile -> - println("${myfile!!.type()} $filename ${"%.2f".format(myfile.size / 1000.0 / 1000.0)} Mbytes") - var countChunks = 0 - if (varname != null) { - val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") - countChunks += testOneVarConcurrent(myfile, myvar) - } else { - myfile.rootGroup().allVariables().forEach { it -> - if (it.datatype.isNumber) { - countChunks += testOneVarConcurrent(myfile, it) - } - } - } - if (countChunks > 0) { - println("${myfile.type()} $filename ${"%.2f".format(myfile.size / 1000.0 / 1000.0)} Mbytes chunks = $countChunks") - } - } - } - - fun testOneVarConcurrent(myFile: Netchdf, myvar: Variable<*>) : Int { - val filename = myFile.location().substringAfterLast('/') - sum = AtomicDouble(0.0) - var countChunks = 0 - val time1 = measureNanoTime { - val chunkIter = myFile.chunkIterator(myvar) - if (chunkIter == null) { - return 0 - } - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array) - countChunks++ - } - } - val sum1 = sum.get() - Stats.of("serialSum", filename, "chunk").accum(time1, countChunks) - - sum.set(0.0) - val time2 = measureNanoTime { - myFile.chunkConcurrent(myvar, null) { sumValues(it.array) } - } - val sum2 = sum.get() - Stats.of("concurrentSum", filename, "chunk").accum(time2, countChunks) - - sum.set(0.0) - val time3 = measureNanoTime { - val arrayData = myFile.readArrayData(myvar, null) - sumValues(arrayData) - } - val sum3 = sum.get() - Stats.of("regularSum", filename, "chunk").accum(time3, countChunks) - - /* if (sum1.isFinite() && sum2.isFinite() && sum3.isFinite()) { - assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2") - assertTrue(nearlyEquals(sum1, sum3), "$sum1 != $sum3 sum3") - } - - */ - return countChunks - } - - var sum = AtomicDouble(0.0) - fun sumValues(array : ArrayTyped<*>) { - if (!array.datatype.isNumber or true) return - for (value in array) { - val number = (value as Number) - val numberd : Double = number.toDouble() - if (numberd.isFinite()) { - sum.getAndAdd(numberd) - } - } - } - -} \ No newline at end of file diff --git a/testfiles/build.gradle.kts b/testfiles/build.gradle.kts index ae7525ea..249031ed 100644 --- a/testfiles/build.gradle.kts +++ b/testfiles/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { api(project(":core")) - implementation(project(":cli")) + // implementation(project(":cli")) implementation(libs.jhdf) implementation(libs.kotlinx.coroutines.core) diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt index ffd617b1..d5998228 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt @@ -59,7 +59,6 @@ class CountVersions { @Test fun countVersions() { - files().forEach { filename -> try { openNetchdfFile(filename).use { ncfile -> diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt index bd13d6b4..65cc8113 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt @@ -5,7 +5,6 @@ import com.sunya.netchdf.NetchdfFileFormat import com.sunya.netchdf.openNetchdfFile import com.sunya.netchdf.openNetchdfFileWithFormat import com.sunya.netchdf.testfiles.H4Files -import com.sunya.netchdf.testutils.Stats import com.sunya.netchdf.testutils.readNetchdfData import com.sunya.netchdf.testutils.testData import org.junit.jupiter.params.ParameterizedTest @@ -15,21 +14,10 @@ import kotlin.test.assertEquals class H4readTest { - init { - Stats.Companion.clear() // problem with concurrent tests - } - companion object { @JvmStatic fun files(): Iterator { - return H4Files.Companion.files() - } - - fun afterAll() { - if (versions.size > 0) { - versions.keys.forEach{ println("$it = ${versions[it]!!.size } files") } - } - Stats.Companion.show() + return H4Files.files() } private val versions = mutableMapOf>() @@ -37,7 +25,9 @@ class H4readTest { @Test fun problem() { - readOneH4header(testData + "devcdm/hdfeos2/MISR_AM1_GP_GMP_P040_O003734_05.eos") + val filename = testData + "devcdm/hdfeos2/MISR_AM1_GP_GMP_P040_O003734_05.eos" + readOneH4header(filename) + readNetchdfData(filename, null, null, true) } @Test @@ -95,14 +85,6 @@ class H4readTest { } } - // not needed - done in netchdfTest - // 2 hours with, 1 hour 12 min without (luckily get to divide by 24 for wall clock) - // @Test - fun readData(filename: String) { - readNetchdfData(filename, null, null, true) - println() - } - // somehow this isnt working anymore alltags not getting set ?? //@Test @OptIn(InternalLibraryApi::class) @@ -120,10 +102,4 @@ class H4readTest { } } - /* takes too long : 47 hours with, 2 hours without (luckily get to divide by 24 for wall clock) - // @Test - fun testReadIterate(filename: String) { - compareNetchIterate(filename) - } */ - } \ No newline at end of file diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt index aa721ed6..e87cb8c8 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt @@ -8,8 +8,9 @@ import com.sunya.netchdf.testfiles.H5Files import com.sunya.netchdf.testutils.AtomicDouble import com.sunya.netchdf.testutils.Stats import com.sunya.netchdf.testutils.compareNetchIterate -import com.sunya.netchdf.testutils.measureNanoTime +import com.sunya.netchdf.testutils.testData import kotlin.collections.iterator +import kotlin.system.measureNanoTime import kotlin.test.* @@ -23,6 +24,11 @@ class H5readConcurrentTest { } } + @Test + fun problemWrf() { + readH5concurrent(testData + "cdmUnitTest/formats/hdf5/wrf/wrf_out_par.h5") + } + @Test fun testReadIterate() { files().forEach { filename -> diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt index c983e8e5..468181dd 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt @@ -1,7 +1,6 @@ package com.sunya.netchdf.hdf5 import com.sunya.netchdf.testfiles.H5Files -import com.sunya.netchdf.testutils.Stats import com.sunya.netchdf.testutils.readNetchdfData import com.sunya.netchdf.testutils.testData import org.junit.jupiter.params.ParameterizedTest @@ -12,19 +11,11 @@ import kotlin.test.* // Sanity check read Hdf5File header, for non-netcdf4 files class H5readTest { - init { - Stats.clear() // problem with concurrent tests - } - companion object { @JvmStatic fun files(): Iterator { return H5Files.files() } - - fun afterAll() { - Stats.show() - } } @Test diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTestJvm.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTestJvm.kt index bc97f882..dacf170f 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTestJvm.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTestJvm.kt @@ -1,7 +1,5 @@ package com.sunya.netchdf.hdf5 -import com.sunya.netchdf.NetchdfFileFormat -import com.sunya.netchdf.openNetchdfFileWithFormat import com.sunya.netchdf.testfiles.H5Files import com.sunya.netchdf.testutils.readNetchdfData import kotlin.test.Test @@ -32,24 +30,4 @@ class H5readTestJvm { readNetchdfData(filename) } } - - fun openH5(filename: String, varname : String? = null) { - println("=================") - println(filename) - openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF5).use { h5file -> - if (h5file == null) { - println("Cant open $filename") - } else { - println(h5file.cdl()) - h5file.rootGroup().allVariables().forEach { println(" ${it.fullname()}") } - - if (varname != null) { - val h5var = h5file.rootGroup().allVariables().find { it.fullname() == varname } - ?: throw RuntimeException("cant find $varname") - val h5data = h5file.readArrayData(h5var) - println(" $varname = $h5data") - } - } - } - } } \ No newline at end of file diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt index afcfd826..32a0da0d 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt @@ -3,7 +3,7 @@ package com.sunya.netchdf.jhdf import com.sunya.cdm.api.Datatype import com.sunya.cdm.api.EnumTypedef import com.sunya.cdm.array.* -import com.sunya.netchdf.hdf5.* +// import com.sunya.netchdf.hdf5filters.* import com.sunya.netchdf.openNetchdfFile import com.sunya.netchdf.testfiles.JhdfFiles import com.sunya.netchdf.testfiles.jhdfTestDir @@ -23,11 +23,12 @@ class JhdfCompare { } } + /* init { FilterRegistrar.registerFilter(Lz4Filter()) FilterRegistrar.registerFilter(LzfFilter()) FilterRegistrar.registerFilter(BitShuffleFilter()) - } + } */ @Test fun compareOneWithJhdf() { diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfShow.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfShow.kt index 0d729405..0ebcd812 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfShow.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfShow.kt @@ -3,6 +3,7 @@ package com.sunya.netchdf.jhdf import com.sunya.cdm.util.Indent import com.sunya.netchdf.hdf5.* +// import com.sunya.netchdf.hdf5filters.* import com.sunya.netchdf.testfiles.JhdfFiles import com.sunya.netchdf.testfiles.jhdfTestDir import com.sunya.netchdf.testutils.readNetchdfData @@ -27,11 +28,11 @@ class JhdfShow { } } - init { +/* init { FilterRegistrar.registerFilter(Lz4Filter()) FilterRegistrar.registerFilter(LzfFilter()) FilterRegistrar.registerFilter(BitShuffleFilter()) - } + } */ @Test fun showJhdfData() { diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/netcdf4/N4charTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/netcdf4/N4charTest.kt index afe8878f..ea9317ff 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/netcdf4/N4charTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/netcdf4/N4charTest.kt @@ -1,6 +1,7 @@ package com.sunya.netchdf.netcdf4 import com.sunya.cdm.api.Datatype +import com.sunya.cdm.array.ArrayString import com.sunya.cdm.array.ArrayUByte import com.sunya.cdm.array.makeStringsFromBytes import com.sunya.netchdf.openNetchdfFile @@ -33,7 +34,7 @@ class N4charTest { } @Test - fun testCharVariable() { + fun testCharVariableIsString() { val filename = testData + "cdmUnitTest/formats/netcdf4/multiDimscale.nc4" openNetchdfFile(filename).use { myfile -> println("--- ${myfile!!.type()} $filename ") @@ -42,20 +43,11 @@ class N4charTest { assertEquals(Datatype.Companion.CHAR, v.datatype) val data = myfile.readArrayData(v) println("file_date data = $data") - assertEquals(Datatype.Companion.CHAR, data.datatype) - assertIs(data) - - val expect = listOf(50,48,49,48,45,48,52,45,48,52,84,48,57,58,49,55,58,52,52,46,48,48,48,48,48,48,50,48,49,48,45,48,52,45,48,52,84,48,57,58,53,52,58,49) - data.forEachIndexed { idx, it -> - if (idx < expect.size) { - assertEquals(expect[idx], it.toInt()) - } - } - val svalues = data.makeStringsFromBytes() - println("svalues = $svalues") + assertEquals(Datatype.Companion.STRING, data.datatype) + assertIs(data) val expectNames = listOf("2010-04-04T09:17:44.000000","2010-04-04T09:54:18.900000","2010-04-04T09:55:51.700000","2010-04-04T09:57:23.700000","2010-04-04T09:58:56.000000","2010-04-04T10:00:29.000000","2010-04-04T10:02:01.200000","2010-04-04T10:03:33.200000","2010-04-04T10:05:09.500000","2010-04-04T10:06:41.600000") - svalues.forEachIndexed { idx, it -> + data.forEachIndexed { idx, it -> assertEquals(expectNames[idx], it) } } diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt index 8b9c48c9..f12d03ea 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt @@ -5,6 +5,7 @@ import com.sunya.cdm.array.* import com.sunya.cdm.util.nearlyEquals import com.sunya.netchdf.openNetchdfFile import kotlin.collections.iterator +import kotlin.system.measureNanoTime import kotlin.test.assertTrue ////////////////////////////////////////////////////////////////////////////////////// @@ -163,8 +164,5 @@ private fun sumValues(array : ArrayTyped<*>, sum : AtomicDouble) { } } -fun measureNanoTime (block: () -> Unit): Long { - return 0 -}