diff --git a/Readme.md b/Readme.md index 94361a64..41de28dd 100644 --- a/Readme.md +++ b/Readme.md @@ -1,5 +1,5 @@ # netchdf -_last updated: 7/18/2025_ +_last updated: 7/22/2025_ This is a rewrite in Kotlin of parts of the devcdm and netcdf-java libraries. diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index 4f6a0aa9..f61d602d 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -3,16 +3,16 @@ } dependencies { - api(project(":core")) + implementation(project(":core")) implementation(libs.lzf) implementation(libs.lz4) - implementation(libs.kotlinx.cli) + // implementation(libs.kotlinx.cli) implementation(libs.oshai.logging) implementation(libs.logback.classic) testImplementation(kotlin("test")) - testImplementation(libs.junit.jupiter.params) + // testImplementation(libs.junit.jupiter.params) } kotlin { diff --git a/cli/src/main/kotlin/com/sunya/netchdf/cli/ncdump.kt b/cli/src/main/kotlin/com/sunya/netchdf/cli/ncdump.kt index a135bd82..a7f6e2a0 100644 --- a/cli/src/main/kotlin/com/sunya/netchdf/cli/ncdump.kt +++ b/cli/src/main/kotlin/com/sunya/netchdf/cli/ncdump.kt @@ -1,8 +1,9 @@ package com.sunya.netchdf.cli -import com.sunya.netchdf.openNetchdfFile -import kotlinx.cli.* +// import com.sunya.netchdf.openNetchdfFile +// import kotlinx.cli.* +/* object ncdump { @JvmStatic fun main(args: Array) { @@ -24,4 +25,6 @@ object ncdump { println(myfile.cdl()) } } -} \ No newline at end of file +} + + */ \ No newline at end of file diff --git a/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt b/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt index aa874582..5fc210dd 100644 --- a/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt +++ b/cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt @@ -1,8 +1,8 @@ package com.sunya.netchdf.hdf5 import com.sunya.cdm.api.computeSize -import kotlin.test.Test import com.sunya.netchdf.openNetchdfFile +import kotlin.test.Test class TestFilters { init { diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt b/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt index 11d2c4df..603af95f 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt @@ -169,3 +169,17 @@ fun convertFromBytes(datatype : Datatype<*>, ba: ByteArray, isBE: Boolean, chars else -> throw IllegalArgumentException("datatype ${datatype}") } } + + +fun Long.reverseByteOrder(wantBE: Boolean): Long { + val b1 = (this and 0xff).toByte() + val b2 = ((this ushr 8) and 0xff).toByte() + val b3 = ((this ushr 16) and 0xff).toByte() + val b4 = ((this ushr 24) and 0xff).toByte() + val b5 = ((this ushr 32) and 0xff).toByte() + val b6 = ((this ushr 40) and 0xff).toByte() + val b7 = ((this ushr 48) and 0xff).toByte() + val b8 = ((this ushr 56) and 0xff).toByte() + + return convertToLong(byteArrayOf(b8, b7, b6, b5, b4, b3, b2, b1), 0, wantBE) +} diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/util/IO.kt b/core/src/commonMain/kotlin/com/sunya/cdm/util/IO.kt index c77071e2..3cff820d 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/util/IO.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/util/IO.kt @@ -30,15 +30,3 @@ internal fun IOcopyB(input: InputStream, out: OutputStream, bufferSize: Int): Lo out.flush() return totalBytesRead } - -/* -fun Long.reverseByteOrder(): Long { - return ByteBuffer.allocate(Long.SIZE_BYTES) - .order(ByteOrder.LITTLE_ENDIAN) - .putLong(this) - .rewind() - .order(ByteOrder.BIG_ENDIAN) - .long -} - - */ \ No newline at end of file diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt index 50b71ebc..1b3f5978 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/FilterPipeline.kt @@ -40,16 +40,6 @@ internal class FilterPipeline( val isBE: Boolean ) { - init { - if (mfp != null) { - mfp.filters.forEach { filter -> - if (filter.filterType == FilterType.lz4) { - println("GOT LZ4!") - } - } - } - } - fun apply(encodedData: ByteArray, filterMask: Int): ByteArray { if (mfp == null) return encodedData var data = encodedData diff --git a/core/src/commonTest/data/hdf5/cenum.h5 b/core/src/commonTest/data/hdf5/cenum.h5 new file mode 100755 index 00000000..d8d92659 Binary files /dev/null and b/core/src/commonTest/data/hdf5/cenum.h5 differ diff --git a/core/src/commonTest/data/hdf5/enum.h5 b/core/src/commonTest/data/hdf5/enum.h5 new file mode 100755 index 00000000..a1e009d1 Binary files /dev/null and b/core/src/commonTest/data/hdf5/enum.h5 differ diff --git a/core/src/commonTest/data/hdf5/enumcmpnd.h5 b/core/src/commonTest/data/hdf5/enumcmpnd.h5 new file mode 100755 index 00000000..5a3d880c Binary files /dev/null and b/core/src/commonTest/data/hdf5/enumcmpnd.h5 differ diff --git a/core/src/commonTest/data/netcdf3/jays_DOMAIN000.nc b/core/src/commonTest/data/netcdf3/jays_DOMAIN000.nc new file mode 100644 index 00000000..e7f423c3 Binary files /dev/null and b/core/src/commonTest/data/netcdf3/jays_DOMAIN000.nc differ diff --git a/core/src/commonTest/data/netcdf3/simple_xy.nc b/core/src/commonTest/data/netcdf3/simple_xy.nc new file mode 100755 index 00000000..010c0a95 Binary files /dev/null and b/core/src/commonTest/data/netcdf3/simple_xy.nc differ diff --git a/core/src/commonTest/data/netcdf4/test_enum_type.nc b/core/src/commonTest/data/netcdf4/test_enum_type.nc new file mode 100755 index 00000000..88c7a7b5 Binary files /dev/null and b/core/src/commonTest/data/netcdf4/test_enum_type.nc differ diff --git a/core/src/commonTest/data/netcdf4/tst_enums.nc b/core/src/commonTest/data/netcdf4/tst_enums.nc new file mode 100755 index 00000000..c8985c99 Binary files /dev/null and b/core/src/commonTest/data/netcdf4/tst_enums.nc differ diff --git a/core/src/commonTest/data/p256.zip b/core/src/commonTest/data/p256.zip new file mode 100644 index 00000000..f162fdc1 Binary files /dev/null and b/core/src/commonTest/data/p256.zip differ diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt index 869db39c..4da8f966 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt @@ -13,101 +13,6 @@ class JhdfReadTest { } } - @Test - fun problem() { - val filename = "$testData/jhdf/compound_datasets_earliest.hdf5" - println(filename) - readNetchdfData(filename, "vlen_chunked_compound", null, true, true) - } - - // HDF5 "$testData/jhdf/bitshuffle_datasets.hdf5" { - //GROUP "/" { - // DATASET "float32_bs0_comp0" { - // DATATYPE H5T_IEEE_F32LE - // DATASPACE SIMPLE { ( 20 ) / ( 20 ) } - // } - // DATASET "float32_bs0_comp2" { - // DATATYPE H5T_IEEE_F32LE - // DATASPACE SIMPLE { ( 20 ) / ( 20 ) } - // } ... - // @Test - fun testBitShuffle() { // 3 - val filename = "$testData/jhdf/bitshuffle_datasets.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - - @Test - fun testBTree2() { // 4 - val filename = "$testData/jhdf/chunked_v4_datasets.hdf5 " - println(filename) - readNetchdfData(filename, "/filtered_btree_v2/int8", null, false, true) - readNetchdfData(filename, "/filtered_extensible_array/int8", null, false, true) - readNetchdfData(filename, "/filtered_extensible_array/large_int16", null, false, true) - readNetchdfData(filename, null, null, true, true) - } - - // getHeapDataAsArray datatype=vlen null - @Test - fun testVlenHeapData() { // 6 - val filename = "$testData/jhdf/compound_datasets_earliest.hdf5" - println(filename) - readNetchdfData(filename, "vlen_chunked_compound", null, true, false) - } - - // see jhdf/src/main/java/io/jhdf/object/datatype/FloatingPoint.java - @Test - fun testHalfFloat() { // 12 - val filename = "$testData/jhdf/float_special_values_earliest.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - - // private typedefs - @Test - fun testCompoundTypedef() { // 18 - val filename = "$testData/jhdf/isssue-523.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - - @Test - fun testFractalHeap() { // 28 - val filename = "$testData/jhdf/test_attribute_latest.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - - // Unknown filter type= lzf name = lzf - // @Test - fun testLzfFilter() { // 37 - val filename = "$testData/jhdf/test_compressed_chunked_datasets_earliest.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - - // private typedefs - @Test - fun testVlen() { // 63 - val filename = "$testData/jhdf/test_vlen_datasets_latest.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - - @Test - fun testCompoundAttribute() { - val filename = "$testData/jhdf/test_compound_scalar_attribute.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - - @Test - fun testCommittedDatatype() { - val filename = "$testData/jhdf/issue255_example.hdf5" - println(filename) - readNetchdfData(filename, null, null, true, false) - } - ///////////////////////////////////// @Test fun testReadAllJhdfFiles() { @@ -115,7 +20,6 @@ class JhdfReadTest { println(filename) readNetchdfData(filename, null, null, true, false) } - } } \ No newline at end of file diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt index e2b570e5..7c904704 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt @@ -4,7 +4,7 @@ import okio.FileSystem import okio.Path import okio.Path.Companion.toPath -const val testData = "src/commonTest/data/" +const val testData = "../core/src/commonTest/data/" fun testFilesIn(dirPath: String): TestFiles.SequenceBuilder { return TestFiles.SequenceBuilder(dirPath) diff --git a/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5CTypeInfo.kt b/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5CTypeInfo.kt index 0443bb45..b5030ede 100644 --- a/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5CTypeInfo.kt +++ b/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5CTypeInfo.kt @@ -182,6 +182,7 @@ internal data class H5CTypeInfo(val type_id: Long, val type_class : Int, val ele 1 -> Datatype.ENUM1.withTypedef(typedef) 2 -> Datatype.ENUM2.withTypedef(typedef) 4 -> Datatype.ENUM4.withTypedef(typedef) + 8 -> Datatype.ENUM8.withTypedef(typedef) else -> throw RuntimeException("Bad hdf5 enum type with size= ${this.elemSize}") } } diff --git a/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt b/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt index 50faafd1..2b0bdb1b 100644 --- a/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt +++ b/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt @@ -241,7 +241,10 @@ class H5Cbuilder(val filename: String) { val ltype = H5L_info2_t.`type$get`(linkinfo, 0L) // H5L_type_t if (ltype == H5L_TYPE_HARD()) { val address = H5L_info2_t.u.ofAddress(linkinfo, context.arena) - if (debug) println("${indent}H5L_TYPE_HARD, address=$address") + if (debug) { + val addressp = address.address() + println("${indent}H5L_TYPE_HARD, address=$addressp reverseBE=${addressp.reverseByteOrder(true)} reverseLE=${addressp.reverseByteOrder(false)}") + } } else if (ltype == H5L_TYPE_SOFT()) { val val_size = H5L_info2_t.u.`val_size$get`(linkinfo, 0L) if (debug) println("${indent}H5L_TYPE_SOFT, val_size=$val_size") @@ -278,6 +281,11 @@ class H5Cbuilder(val filename: String) { val loc_id = H5Oopen(group_id, linkname_p, H5P_DEFAULT_LONG) val oinfo_p = H5O_info2_t.allocate(context.arena) + val errno = H5Oget_info3(loc_id, oinfo_p, H5O_INFO_ALL()) + if (errno < 0) { + println("H5Oget_info3 failed ($errno) on linkname $linkname") + return 0 // bogus "no error" + } checkErr("H5Oget_info3", H5Oget_info3(loc_id, oinfo_p, H5O_INFO_ALL())) val otype = H5O_info2_t.`type$get`(oinfo_p) val num_attr = H5O_info2_t.`num_attrs$get`(oinfo_p) @@ -506,13 +514,9 @@ class H5Cbuilder(val filename: String) { if (isVariable) context.group.addVariable(vb) // datasetId is transient - var address = H5Dget_offset(datasetId) - /* println("**H5Cbuilder obj_name=$obj_name datasetId=$datasetId address=$address") // maybe there a byte order problem ?? - if (address < 0) { - val reversed = datasetId.reverseByteOrder() // doesnt work - address = H5Dget_offset(reversed) - println(" try again with byte order reversed: datasetId=$reversed address=$address") // maybe there a byte order problem ?? - } */ + val address = H5Dget_offset(datasetId) + // println("**H5Cbuilder obj_name=$obj_name datasetId=$datasetId address=$address") // maybe there a byte order problem ?? + if (address > 0) datasetMap[address] = Pair(context.group, vb) if (obj_name.startsWith("StructMetadata")) { diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt index 04d3ede6..13873bb1 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt @@ -69,8 +69,9 @@ fun compareGroup(group: Group, cgroup: Group, indent: Indent, showCompare: Boole } group.variables.forEach { v -> - val cv = cgroup.variables.find { it.name == v.name }!! - compareVariable(v, cv, indent.incr(), showCompare) + val cv = cgroup.variables.find { it.name == v.name } + if (cv == null) println("***cant find c variable ${v.fullname()}") + else compareVariable(v, cv, indent.incr(), showCompare) } group.groups.forEach { nested -> diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt index 159a388a..f66c6922 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt @@ -5,6 +5,7 @@ package com.sunya.netchdf import com.sunya.cdm.util.InternalLibraryApi import com.sunya.netchdf.hdf4.Hdf4File import com.sunya.netchdf.hdf5Clib.compareDataWithHdf5Clib +import com.sunya.netchdf.netcdf4.compareH5andNclib import com.sunya.netchdf.testutils.testData import kotlin.test.* import kotlin.test.assertTrue @@ -13,269 +14,22 @@ val problemDir = testData + "/exclude/" class KnownProblemFiles { - // LOOK theory is that a HDF5_DIMENSION_LIST that is a vlen of reference is a netcdf4 file - // but its not, we dont care, but using Nclib fails to detect missing data for Variable Granule. - @Test - fun testConfuseHdf5WithNetcdf4() { - val filename = testData + "/netchdf/knox/SATMS_justdims_npp_d20120619_t1121416_e1122133_b03335_c20120619200237705890_noaa_ops.h5" - // CompareNetchdf(filename, true) - compareDataWithClib(filename, "Granule") - } - - @Test - fun testConfusedHdf5() { - val filename = testData + "/netchdf/knox/SATMS_justdims_npp_d20120619_t1121416_e1122133_b03335_c20120619200237705890_noaa_ops.h5" - // CompareNetchdf(filename, true) - compareDataWithHdf5Clib(filename, "Granule", null) - } - /* tst_grps - Maybe netcdf-4 skips opaque type? - -(Netcdf4) -netcdf tst_grps.nc4 { -types: - int(*) vlen-1 ; - byte(*) vlen-2 ; - -group: the_in_crowd { -} - -group: the_out_crowd { - - group: the_confused_crowd { - } -} -} - -(netchdf) -netcdf tst_grps.nc4 { -types: - opaque(10) opaque-1 ; - int(*) vlen-1 ; - byte(*) vlen-2 ; - -group: the_in_crowd { - types: - opaque(7) opaque-2 ; -} - -group: the_out_crowd { - types: - opaque(4) opaque-3 ; - - group: the_confused_crowd { - types: - opaque(13) opaque-4 ; - } -} -} */ - @Test - fun tst_grps() { - CompareCdmWithClib(testData + "devcdm/netcdf4/tst_grps.nc4") - } - - @Test - // We seem to be suppressing user types that are not used. - // C library shows - // compound compound_att_float { - // float field0 ; - // float field1 ; - // float field2 ; - // float field3 ; - // }; // compound_att_float - fun compoundAttributeTest() { - CompareCdmWithClib(testData + "cdmUnitTest/formats/netcdf4/compound-attribute-test.nc") - } - - // We use HDFEOS_INFORMATION to modify the structure, Nclib does not. - @Test - fun testEos() { - compareDataWithClib(testData + "cdmUnitTest/formats/hdf5/aura/MLS-Aura_L2GP-BrO_v01-52-c01_2007d029.he5") - } - - // NClibFile has: - // group: image9 { - // variables: - // ubyte image_data(3712, 3712) ; - // :DISPLAY_ORIGIN = "UL" ; - // - // Hdf5file has - // group: image9 { - // variables: - // ubyte image_data(3712, 3712) ; - // :CLASS = "IMAGE" ; - // :DISPLAY_ORIGIN = "UL" ; - // :PALETTE = "/visualisation9/color_palette" ; - // - // Im guessing CLASS and PALETTE attributes are supressed. In c library or our code? - - @Test - fun problemReferenceToPallette() { - // compareH5andNclib(testData + "netchdf/esben/level2_MSG2_8bit_VISIR_STD_20091005_0700.H5") - compareDataWithClib(testData + "netchdf/esben/level2_MSG2_8bit_VISIR_STD_20091005_0700.H5") - } - - // netcdf library version 4.9.2-development of Mar 19 2023 10:42:31 - // ~/install/netcdf4/bin:$ ./ncdump -h /home/all/testdata/netchdf/austin/H12007_1m_MLLW_1of6.bag - // netcdf H12007_1m_MLLW_1of6 { - // - //group: BAG_root { - // dimensions: - // phony_dim_0 = 3027 ; - // phony_dim_1 = 6981 ; - // phony_dim_2 = UNLIMITED ; // (4953 currently) - // variables: - // float elevation(phony_dim_0, phony_dim_1) ; - // elevation:Minimum\ Elevation\ Value = -26.52571f ; - // elevation:Maximum\ Elevation\ Value = -4.265297f ; - // char metadata(phony_dim_2) ; - // float uncertainty(phony_dim_0, phony_dim_1) ; - // uncertainty:Minimum\ Uncertainty\ Value = 0.139f ; - // uncertainty:Maximum\ Uncertainty\ Value = 1.001f ; - // - // // group attributes: - // :Bag\ Version = "1.1.0\000\000\000ISO19139/smXML/metadataE" ; - // } // group BAG_root - //} - // - // NClibFile: - // netcdf H12007_1m_MLLW_1of6.bag { - // - // group: BAG_root { - // variables: - // float elevation(3027, 6981) ; - // :Maximum_Elevation_Value = -4.265297f ; - // :Minimum_Elevation_Value = -26.525713f ; - // char metadata(4953) ; - // float uncertainty(3027, 6981) ; - // :Maximum_Uncertainty_Value = 1.001f ; - // :Minimum_Uncertainty_Value = 0.13900001f ; - // - // // group attributes: - // :Bag_Version = "1.1.0" ; - // } - //} - //////////////////////////////////////////////////////////////////////// - - // Upgrade to netcdf library version 4.10.0-development of May 23 2025 14:45:19 $ - // ~/install/netcdf4/bin:$ ./ncdump -h /home/all/testdata/netchdf/austin/H12007_1m_MLLW_1of6.bag - - - //netcdf H12007_1m_MLLW_1of6 { - // - //group: BAG_root { - // types: - // compound _AnonymousCompound1 { - // uint row ; - // uint col ; - // float depth ; - // float uncertainty ; - // ubyte track_code ; - // short list_series ; - // }; // _AnonymousCompound1 - // dimensions: - // phony_dim_0 = 3027 ; - // phony_dim_1 = 6981 ; - // phony_dim_2 = UNLIMITED ; // (4953 currently) - // phony_dim_3 = UNLIMITED ; // (6 currently) - // variables: - // float elevation(phony_dim_0, phony_dim_1) ; - // elevation:Minimum\ Elevation\ Value = -26.52571f ; - // elevation:Maximum\ Elevation\ Value = -4.265297f ; - // char metadata(phony_dim_2) ; - // _AnonymousCompound1 tracking_list(phony_dim_3) ; - // tracking_list:Tracking\ List\ Length = 6U ; - // float uncertainty(phony_dim_0, phony_dim_1) ; - // uncertainty:Minimum\ Uncertainty\ Value = 0.139f ; - // uncertainty:Maximum\ Uncertainty\ Value = 1.001f ; - // - // // group attributes: - // :Bag\ Version = "1.1.0\000\000\000ISO19139/smXML/metadataE" ; - // } // group BAG_root - //} - // - // NClibFile: - // ncfile = netcdf H12007_1m_MLLW_1of6.bag { - // - // group: BAG_root { - // types: - // compound _AnonymousCompound1 { - // uint row ; - // uint col ; - // float depth ; - // float uncertainty ; - // ubyte track_code ; - // short list_series ; - // }; // _AnonymousCompound1 - // variables: - // float elevation(3027, 6981) ; - // :Maximum_Elevation_Value = -4.265297f ; - // :Minimum_Elevation_Value = -26.525713f ; - // char metadata(4953) ; - // _AnonymousCompound1 tracking_list(6) ; - // :Tracking_List_Length = 6 ; - // float uncertainty(3027, 6981) ; - // :Maximum_Uncertainty_Value = 1.001f ; - // :Minimum_Uncertainty_Value = 0.13900001f ; - // - // // group attributes: - // :Bag_Version = "1.1.0" ; - // } - //} - ////////////////////////////////////////// - // org.opentest4j.AssertionFailedError: variable /BAG_root/metadata ==> expected: - // - // ','<','s','m','X','M', ... - //> but was: = pt[i] : String.format("shape[%s]=(%s) should not be less than pt[%s]=(%s)", i, shape[i], i, pt[i]); - // tile[i] = index[i] / chunk[i] // LOOK seems wrong, rounding down ?? - // } - // return tile - // } - /* @Test - fun testNetchIterate() { - val filename = problemDir + "UpperDeschutes_t4p10_swemelt.nc" - val varname = "UpperDeschutes_t4p10_swemelt" - // *** double UpperDeschutes_t4p10_swemelt[8395, 781, 385] skip read ArrayData too many bytes= 2524250575 - compareNetchIterate(filename, varname) - - // readH5concurrent(filename, null) - } */ - - // * IP8/1 usedBy=false pos=18664902/32 rgb=147,0,108,144,0,111,141,0,114,138,0,117,135,0,120,132,0,123,129,0,126,126,0,129,123,0,132,120,0,135,117,0,138,114,0,141,111,0,144,108,0,147,105,0,150,102,0,153,99,0,156,96,0,159,93,0,162,90,0,165,87,0,168,84,0,171,81,0,174,78,0,177,75,0,180,72,0,183,69,0,186,66,0,189,63,0,192,60,0,195,57,0,198,54,0,201,51,0,204,48,0,207,45,0,210,42,0,213,39,0,216,36,0,219,33,0,222,30,0,225,27,0,228,24,0,231,21,0,234,18,0,237,15,0,240,12,0,243,9,0,246,6,0,249,0,0,252,0,0,255,0,5,255,0,10,255,0,16,255,0,21,255,0,26,255,0,32,255,0,37,255,0,42,255,0,48,255,0,53,255,0,58,255,0,64,255,0,69,255,0,74,255,0,80,255,0,85,255,0,90,255,0,96,255,0,101,255,0,106,255,0,112,255,0,117,255,0,122,255,0,128,255,0,133,255,0,138,255,0,144,255,0,149,255,0,154,255,0,160,255,0,165,255,0,170,255,0,176,255,0,181,255,0,186,255,0,192,255,0,197,255,0,202,255,0,208,255,0,213,255,0,218,255,0,224,255,0,229,255,0,234,255,0,240,255,0,245,255,0,250,255,0,255,255,0,255,247,0,255,239,0,255,231,0,255,223,0,255,215,0,255,207,0,255,199,0,255,191,0,255,183,0,255,175,0,255,167,0,255,159,0,255,151,0,255,143,0,255,135,0,255,127,0,255,119,0,255,111,0,255,103,0,255,95,0,255,87,0,255,79,0,255,71,0,255,63,0,255,55,0,255,47,0,255,39,0,255,31,0,255,23,0,255,15,0,255,0,8,255,0,16,255,0,24,255,0,32,255,0,40,255,0,48,255,0,56,255,0,64,255,0,72,255,0,80,255,0,88,255,0,96,255,0,104,255,0,112,255,0,120,255,0,128,255,0,136,255,0,144,255,0,152,255,0,160,255,0,168,255,0,176,255,0,184,255,0,192,255,0,200,255,0,208,255,0,216,255,0,224,255,0,232,255,0,240,255,0,248,255,0,255,255,0,255,251,0,255,247,0,255,243,0,255,239,0,255,235,0,255,231,0,255,227,0,255,223,0,255,219,0,255,215,0,255,211,0,255,207,0,255,203,0,255,199,0,255,195,0,255,191,0,255,187,0,255,183,0,255,179,0,255,175,0,255,171,0,255,167,0,255,163,0,255,159,0,255,155,0,255,151,0,255,147,0,255,143,0,255,139,0,255,135,0,255,131,0,255,127,0,255,123,0,255,119,0,255,115,0,255,111,0,255,107,0,255,103,0,255,99,0,255,95,0,255,91,0,255,87,0,255,83,0,255,79,0,255,75,0,255,71,0,255,67,0,255,63,0,255,59,0,255,55,0,255,51,0,255,47,0,255,43,0,255,39,0,255,35,0,255,31,0,255,27,0,255,23,0,255,19,0,255,15,0,255,11,0,255,7,0,255,3,0,255,0,0,250,0,0,245,0,0,240,0,0,235,0,0,230,0,0,225,0,0,220,0,0,215,0,0,210,0,0,205,0,0,200,0,0,195,0,0,190,0,0,185,0,0,180,0,0,175,0,0,170,0,0,165,0,0,160,0,0,155,0,0,150,0,0,145,0,0,140,0,0,135,0,0,130,0,0,125,0,0,120,0,0,115,0,0,110,0,0,105,0,0,0,0,0, - // * LUT/1 usedBy=false pos=18664902/32 nelems=null - // DIL/1 usedBy=true pos=18665670/31 for=1/201 text=palette + // superblockIsOffsetNPP.h5 is opened as a Netcdf4 file, but same problem with hdf5 + // not picking up missing value + // *** FAIL comparing data for variable = ubyte Granule [Granule] + //255 != 0 at idx = 1 + //255 != 0 at idx = 2 + //255 != 0 at idx = 3 + //255 != 0 at idx = 4 + //255 != 0 at idx = 5 + //255 != 0 at idx = 6 + // *** count values differ = 6 same = 1 @Test - fun problemDIL() { - val filename = testData + "hdf4/S2007329.L3m_DAY_CHLO_9" - openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF4).use { h4file -> - if (h4file == null) { - println("Cant open $filename") - } else { - println("--- ${h4file.type()} $filename ") - val hdf4File = h4file as Hdf4File - assertTrue(0 == hdf4File.header.showTags(true, true, false)) - } - } + fun superblockIsOffsetNPP() { + val filename = problemDir + "superblockIsOffsetNPP.h5" + compareH5andNclib(filename, compareData = true) + CompareCdmWithClib(filename, true) + compareDataWithClib(filename, varname = "Granule") } @Test @@ -283,33 +37,22 @@ group: the_out_crowd { CompareCdmWithClib(problemDir + "OMI-Aura_L2-OMTO3_2009m0829t1219-o27250_v003-2009m0829t175727.he5") } + // H5C cant find dataset reference for Attribute(orgName=PALETTE, datatype=reference, values=[-1]) @Test - fun problem() { + fun attReference() { val filename = problemDir + "RAD_NL25_PCP_NA_200804110600.h5" CompareCdmWithClib(filename) compareDataWithClib(filename) } // tried to add duplicate dimension 'lv1' + // ncdump gives "NetCDF: Invalid dimension ID or name". assumed damaged file @Test - fun problem2() { - val filename = testData + "netchdf/gilmore/data.nc" + fun dupDimensions() { + val filename = problemDir + "gilmore/data.nc" + showNetchdfHeader(filename) CompareCdmWithClib(filename) compareDataWithClib(filename) } - // not picking up missing value ?? - // *** FAIL comparing data for variable = ubyte Granule [Granule] - //255 != 0 at idx = 1 - //255 != 0 at idx = 2 - //255 != 0 at idx = 3 - //255 != 0 at idx = 4 - //255 != 0 at idx = 5 - //255 != 0 at idx = 6 - // *** count values differ = 6 same = 1 - @Test - fun superblockIsOffsetNPP() { - compareDataWithClib(testData + "exclude/superblockIsOffsetNPP.h5") - } - } \ No newline at end of file diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt index cb883aae..8e29580c 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt @@ -88,7 +88,7 @@ class NetchdfClibExtra { } } - @Test + // @Test fun testFilesAfter() { var skip = true files().forEach { filename -> diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfCompareTest.kt similarity index 59% rename from testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt rename to testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfCompareTest.kt index 61721a92..e629b936 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfCompareTest.kt @@ -5,7 +5,7 @@ import com.sunya.netchdf.compareDataWithClib import com.sunya.netchdf.testfiles.JhdfFiles import kotlin.test.Test -class JhdfReadTest { +class JhdfCompareTest { companion object { @JvmStatic fun files(): Iterator { @@ -19,25 +19,29 @@ class JhdfReadTest { FilterRegistrar.registerFilter(BitShuffleFilter()) } */ + // H5Aread VlenString return -1 + //java.io.IOException: H5Aread VlenString return -1 + // at com.sunya.netchdf.hdf5Clib.H5CbuilderKt.checkErr(H5Cbuilder.kt:720) + // at com.sunya.netchdf.hdf5Clib.H5Cbuilder.readVlenStrings$testclibs(H5Cbuilder.kt:645) @Test fun problem() { - CompareCdmWithClib("../core/src/commonTest/data/jhdf/compound_datasets_earliest.hdf5", showCdl = true, showCompare = true) - // CompareNetchdf("../core/src/commonTest/data/jhdf/compound_datasets_earliest.hdf5", showCdl= true) + val filename = "../core/src/commonTest/data/jhdf/globalheaps_test.hdf5" + CompareCdmWithClib(filename, showCdl = true, showCompare = true) + compareDataWithClib(filename) } + // lots of link messages we dont handle @Test fun problem2() { - CompareCdmWithClib("../core/src/commonTest/data/jhdf/compound_datasets_latest.hdf5", showCdl= true, showCompare = true) - } - - @Test - fun failure() { - CompareCdmWithClib("../core/src/commonTest/data/jhdf/globalheaps_test.hdf5", showCdl= true, showCompare = true) + val filename = "../core/src/commonTest/data/jhdf/test_file.hdf5" + CompareCdmWithClib(filename, showCdl = true, showCompare = true) + compareDataWithClib(filename) } @Test fun compareNetchdf() { files().forEach { filename -> + println(filename) try { CompareCdmWithClib(filename, false, false) } catch (e: Throwable) { @@ -50,6 +54,7 @@ class JhdfReadTest { @Test fun compareDataWithClib() { files().forEach { filename -> + println(filename) try { compareDataWithClib(filename) } catch (e: Throwable) { diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt index 2d633c0b..7869c3a1 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt @@ -106,14 +106,6 @@ class H5CopenTest { compareDataWithClib(filename, "IntArray") } - @Test - fun testConfuseHdf5WithNetcdf4() { - val filename = - testData + "/netchdf/knox/SATMS_justdims_npp_d20120619_t1121416_e1122133_b03335_c20120619200237705890_noaa_ops.h5" - // CompareNetchdf(filename, true) - compareDataWithHdf5Clib(filename, "Granule") - } - /////////////////////////////////////////////////////////////////////////////////// @Test diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt index 05f1bbc1..f6e135c0 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt @@ -2,6 +2,7 @@ package com.sunya.netchdf.netcdf4 import com.sunya.netchdf.NetchdfFileFormat import com.sunya.netchdf.compareDataWithClib +import com.sunya.netchdf.compareNetcdfData import com.sunya.netchdf.compareNetchdfCdm import com.sunya.netchdf.netcdfClib.NClibFile import com.sunya.netchdf.openNetchdfFileWithFormat @@ -34,17 +35,19 @@ class CompareH5andNclib { compareH5andNclib(filename) } } - - fun compareH5andNclib(filename: String) { - println("===================================================") - openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF5).use { h5file -> - println("${h5file!!.type()} $filename ") - println("\n${h5file.cdl()}") - - NClibFile(filename).use { nclibfile -> - println("NClibFile = ${nclibfile.cdl()}") - compareNetchdfCdm(nclibfile, h5file) - } +} + +fun compareH5andNclib(filename: String, compareData : Boolean = false) { + println("===================================================") + openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF5).use { h5file -> + println("Opened as ${h5file!!.javaClass.simpleName}") + println("${h5file!!.type()} $filename ") + println("\n${h5file.cdl()}") + + NClibFile(filename).use { nclibfile -> + println("NClibFile = ${nclibfile.cdl()}") + compareNetchdfCdm(nclibfile, h5file) + compareNetcdfData(h5file, nclibfile, varname = null) } } } \ No newline at end of file diff --git a/testfiles/src/main/kotlin/com/sunya/netchdf/testfiles/JhdfFiles.kt b/testfiles/src/main/kotlin/com/sunya/netchdf/testfiles/JhdfFiles.kt index 5e0df953..efb8e269 100644 --- a/testfiles/src/main/kotlin/com/sunya/netchdf/testfiles/JhdfFiles.kt +++ b/testfiles/src/main/kotlin/com/sunya/netchdf/testfiles/JhdfFiles.kt @@ -16,6 +16,9 @@ class JhdfFiles { .addNameFilter { name -> !name.endsWith("test_chunked_datasets_earliest.hdf5") } .addNameFilter { name -> !name.endsWith("test_chunked_datasets_latest.hdf5") } .addNameFilter { name -> !name.endsWith("external_link.hdf5") } + .addNameFilter { name -> !name.endsWith("globalheaps_test.hdf5") } + .addNameFilter { name -> !name.endsWith("test_file.hdf5") } + .addNameFilter { name -> !name.endsWith("test_byteshuffle_compressed_datasets_latest.hdf5") } .build() return jhdf.iterator() } diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt index 9a326465..afcfd826 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/jhdf/JhdfCompare.kt @@ -1,6 +1,5 @@ package com.sunya.netchdf.jhdf - import com.sunya.cdm.api.Datatype import com.sunya.cdm.api.EnumTypedef import com.sunya.cdm.array.* @@ -12,8 +11,6 @@ import com.sunya.netchdf.testutils.readNetchdfData import com.sunya.netchdf.testutils.testData import io.jhdf.HdfFile import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource import java.nio.file.Paths import kotlin.test.Test import kotlin.test.assertEquals @@ -32,14 +29,6 @@ class JhdfCompare { FilterRegistrar.registerFilter(BitShuffleFilter()) } - @Test - fun compareDataWithJhdf() { - files().forEach { filename -> - println("=============================") - compareDataWithJhdf(filename) - } - } - @Test fun compareOneWithJhdf() { // this ones pretty bad @@ -54,6 +43,23 @@ class JhdfCompare { compareDataWithJhdf(testData + "netcdf-c_hdf5_superblocks/netcdf-c-test-files/v1_8/nc_test4__tst_xplatform2_3.nc", null, true, true) } + ///////////////////////////////////// + @Test + fun readAllData() { + files().forEach { filename -> + println(filename) + readNetchdfData(filename, null, null, false, false) + } + } + + @Test + fun compareDataWithJhdf() { + files().forEach { filename -> + println("=============================") + compareDataWithJhdf(filename) + } + } + fun compareDataWithJhdf(filename: String, varname: String? = null, showData: Boolean = false, showCdl: Boolean = false) { println(filename) println(varname) @@ -274,15 +280,6 @@ class JhdfCompare { } } - - ///////////////////////////////////// - @ParameterizedTest - @MethodSource("files") - fun testReadN3data(filename: String) { - println(filename) - readNetchdfData(filename, null, null, true, false) - } - } // double iterator (iterator of iterator)