From 0def928ac52763172db4794b3d726d69ab226f96 Mon Sep 17 00:00:00 2001 From: JohnLCaron Date: Tue, 22 Jul 2025 07:49:25 -0600 Subject: [PATCH] change CHAR datatype return: STRING vs UBYTE. readBtreeVer1 missing data bugfix. hdf5 shared datatypes. CompareNetchdf -> CompareCdmWithClib --- Readme.md | 19 ++- .../com/sunya/cdm/array/StructureMember.kt | 2 +- .../com/sunya/cdm/array/TypeConverter.kt | 2 +- .../com/sunya/cdm/array/TypedByteArray.kt | 8 +- .../kotlin/com/sunya/netchdf/hdf4/Tag.kt | 11 -- .../com/sunya/netchdf/hdf5/H5chunkReader.kt | 28 ++-- .../kotlin/com/sunya/netchdf/hdf5/H5group.kt | 3 +- .../kotlin/com/sunya/netchdf/hdf5/H5reader.kt | 2 +- .../kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt | 7 +- .../com/sunya/netchdf/testutil/TestFiles.kt | 2 +- .../com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt | 4 +- .../com/sunya/netchdf/netcdfClib/NClibFile.kt | 67 ++++---- .../com/sunya/netchdf/netcdfClib/UserTypes.kt | 15 +- .../com/sunya/netchdf/CompareCdmWithClib.kt | 158 ++++++++++++++++++ .../com/sunya/netchdf/CompareNetchdf.kt | 158 ------------------ .../com/sunya/netchdf/KnownProblemFiles.kt | 14 +- .../com/sunya/netchdf/NetchdfClibExtra.kt | 23 +-- .../com/sunya/netchdf/NetchdfClibTest.kt | 50 ++++-- .../test/kotlin/com/sunya/netchdf/TestNPP.kt | 2 +- .../com/sunya/netchdf/hdf4/H4Ccompare.kt | 8 - .../com/sunya/netchdf/hdf5/Hdf5Compare.kt | 51 ++---- .../com/sunya/netchdf/hdf5/Hdf5openTest.kt | 14 +- .../com/sunya/netchdf/hdf5/JhdfReadTest.kt | 12 +- .../com/sunya/netchdf/hdf5Clib/H5CopenTest.kt | 28 +--- .../sunya/netchdf/netcdf3/N3dataCompare.kt | 20 --- .../netchdf/netcdf4/CompareH5andNclib.kt | 50 ++++++ 26 files changed, 370 insertions(+), 388 deletions(-) create mode 100644 testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt delete mode 100644 testclibs/src/test/kotlin/com/sunya/netchdf/CompareNetchdf.kt create mode 100644 testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt diff --git a/Readme.md b/Readme.md index 9a0913d3..94361a64 100644 --- a/Readme.md +++ b/Readme.md @@ -256,16 +256,21 @@ For example, a Variable of datatype Float will return an ArrayFloat, which is Ar #### Datatype * _Datatype.ENUM_ returns an array of the corresponding UBYTE/USHORT/UINT. Call _data.convertEnums()_ to turn this into an ArrayString of corresponding enum names. -* _Datatype.CHAR_: All Attributes of type CHAR are assumed to be Strings. All Variables of type CHAR return data as - ArrayUByte. Call _data.makeStringsFromBytes()_ to turn this into Strings with the array rank reduced by one. - * Netcdf-3 does not have STRING or UBYTE types. In practice, CHAR is used for either. - * Netcdf-4/HDF5 library encodes CHAR values as HDF5 string type with elemSize = 1, so we use that convention to detect - legacy CHAR variables in HDF5 files. (NC_CHAR should not be used in Netcdf-4, use NC_UBYTE or NC_STRING.) +* CHAR vs STRING: + * Attributes of type CHAR are always assumed to be Strings. + * Netcdf-3 does not have STRING or UBYTE types, and in practice, CHAR is used for either. Variables of type CHAR + return data as ArrayUByte. + * Netcdf-4 encodes CHAR values as HDF5 string type with elemSize = 1, so we use that convention to detect + legacy CHAR variables in HDF5 format. (NC_CHAR should not be used in new Netcdf-4 files, use NC_UBYTE or NC_STRING.) + Variables of type CHAR return data as STRING, since users can use UBYTE if thats what they intend. * Netcdf-4/HDF5 String variables may be fixed or variable length. For fixed Strings, we set the size of Datatype.STRING to - the fixed size. For both fixed and variable length Strings, the string withh be truncated at the first zero byte, if any. + the fixed size. For both fixed and variable length Strings, the string will be truncated at the first zero byte, if any. * HDF4 does not have a STRING type, but does have signed and unsigned CHAR, and signed and unsigned BYTE. - We map both signed and unsigned to Datatype.CHAR and handle it as above (Attributes are Strings, Variables are UBytes). + Both signed and unsigned CHAR are mapped to Datatype.CHAR, whose data is returned as Strings for Attributes, + and ArrayUByte for Variables. + * Call _data.makeStringsFromBytes() to turn ArrayUByte into ArrayString with the array reduced by one. * _Datatype.STRING_ always appears to be variable length to the user, regardless of whether the data in the file is variable or fixed length. +* _Datatype.STRING_ is always encoded as UTF8. TODO add option to change encoding, probably when opening the file. #### Typedef Unlike Netcdf-Java, we follow Netcdf-4 "user defined types" and add typedefs for Compound, Enum, Opaque, and Vlen. diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt b/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt index 7e0d294b..5ca075b5 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/array/StructureMember.kt @@ -24,7 +24,7 @@ class StructureMember(orgName: String, val datatype : Datatype, val offset if (nelems > 1 && !datatype.isVlenString && (datatype != Datatype.VLEN)) { val tba = TypedByteArray(this.datatype, sdata.ba, offset, this.isBE) - return tba.convertToArrayTyped(shape) + return tba.convertToArrayTyped(shape) // TODO charToString = ?? } val enumTypedef = if (!datatype.isEnum || this.datatype.typedef == null) null diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt b/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt index 8bc0df12..11d2c4df 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/array/TypeConverter.kt @@ -135,7 +135,7 @@ fun makeString(ba: ByteArray) = ba.decodeToString(charset = Charsets.UTF8) // needed when setting fillValue from Attribute value fun convertToBytes(datatype : Datatype<*>, value: Any?, isBE: Boolean, charset : Charset = Charsets.UTF8): ByteArray { - if ( value == null) return ByteArray(datatype.size) + if (value == null) return ByteArray(datatype.size) return when (value) { is Byte -> byteArrayOf(value) // is Char -> byteArrayOf(value.toByte()) // avoid CHAR altogether diff --git a/core/src/commonMain/kotlin/com/sunya/cdm/array/TypedByteArray.kt b/core/src/commonMain/kotlin/com/sunya/cdm/array/TypedByteArray.kt index a4e0cf61..dabeb9f2 100644 --- a/core/src/commonMain/kotlin/com/sunya/cdm/array/TypedByteArray.kt +++ b/core/src/commonMain/kotlin/com/sunya/cdm/array/TypedByteArray.kt @@ -77,11 +77,15 @@ class TypedByteArray(val datatype: Datatype, val ba: ByteArray, val offset } } - fun convertToArrayTyped(shape: IntArray, elemSize : Int? = null): ArrayTyped { + fun convertToArrayTyped(shape: IntArray, elemSize : Int? = null, charToString : Boolean = false): ArrayTyped { val nelems = shape.computeSize() val result = when (datatype) { Datatype.BYTE -> ArrayByte(shape, ByteArray(nelems) { this.get(it) as Byte } ) - Datatype.CHAR, Datatype.UBYTE, Datatype.ENUM1 -> ArrayUByte(shape, datatype, UByteArray(nelems) { this.get(it) as UByte }) + Datatype.UBYTE, Datatype.ENUM1 -> ArrayUByte(shape, datatype, UByteArray(nelems) { this.get(it) as UByte }) + Datatype.CHAR -> { + val ubytes = ArrayUByte(shape, datatype, UByteArray(nelems) { this.get(it) as UByte }) + if (charToString) ubytes.makeStringsFromBytes() else ubytes + } Datatype.SHORT -> ArrayShort(shape, ShortArray(nelems) { this.get(it) as Short }) Datatype.USHORT, Datatype.ENUM2 -> ArrayUShort(shape, datatype, UShortArray(nelems) { this.get(it) as UShort }) Datatype.INT -> ArrayInt(shape, IntArray(nelems) { this.get(it) as Int } ) diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt index 05a492db..cc23d685 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/Tag.kt @@ -378,17 +378,6 @@ internal class TagRasterImage(icode: Int, refno: Int, offset : Long, length : In val shape = intArrayOf(tagID.ydim, tagID.xdim, tagID.nelems) val tba = TypedByteArray(datatype, raw, 0, isBE = true) raster = tba.convertToArrayTyped(shape) - - /* - raster = when (datatype) { - Datatype.BYTE -> ArrayByte(shape, bb) - Datatype.UBYTE -> ArrayUByte(shape, bb) - Datatype.SHORT -> ArrayShort(shape, bb) - Datatype.USHORT -> ArrayUShort(shape, bb) - Datatype.INT -> ArrayInt(shape, bb) - Datatype.UINT -> ArrayUInt(shape, bb) - else -> throw RuntimeException("not supporting $datatype for TagRasterImage") - } */ } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt index 35805738..d5d0d73d 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5chunkReader.kt @@ -85,21 +85,19 @@ internal class H5chunkReader(val h5 : H5builder) { var transferChunks = 0 val state = OpenFileState(0L, vinfo.h5type.isBE) for (dataChunk: DataChunkIF in tiledData.dataChunks(wantSpace)) { // : Iterable - if (!dataChunk.isMissing()) { // TODO fill value - val dataSection = IndexSpace(v2.rank, dataChunk.offsets(), vinfo.storageDims) - val chunker = Chunker(dataSection, wantSpace) // each DataChunkEntry has its own Chunker iteration - if (dataChunk.isMissing()) { - if (debugChunking) println(" missing ${dataChunk.show(tiledData.tiling)}") - chunker.transferMissing(vinfo.fillValue, elemSize, ba) - } else { - if (debugChunking) println(" chunk=${dataChunk.show(tiledData.tiling)}") - state.pos = dataChunk.childAddress() - val chunkData = h5.raf.readByteArray(state, dataChunk.chunkSize()) - val filteredData = if (dataChunk.filterMask() == null) chunkData - else filters.apply(chunkData, dataChunk.filterMask()!!) - chunker.transferBA(filteredData, 0, elemSize, ba, 0) - transferChunks += chunker.transferChunks - } + val dataSection = IndexSpace(v2.rank, dataChunk.offsets(), vinfo.storageDims) + val chunker = Chunker(dataSection, wantSpace) // each DataChunkEntry has its own Chunker iteration + if (dataChunk.isMissing()) { + if (debugChunking) println(" missing ${dataChunk.show(tiledData.tiling)}") + chunker.transferMissing(vinfo.fillValue, elemSize, ba) + } else { + if (debugChunking) println(" chunk=${dataChunk.show(tiledData.tiling)}") + state.pos = dataChunk.childAddress() + val chunkData = h5.raf.readByteArray(state, dataChunk.chunkSize()) + val filteredData = if (dataChunk.filterMask() == null) chunkData + else filters.apply(chunkData, dataChunk.filterMask()!!) + chunker.transferBA(filteredData, 0, elemSize, ba, 0) + transferChunks += chunker.transferChunks } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt index dbea8389..6e65ccf6 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5group.kt @@ -285,9 +285,10 @@ internal class H5GroupBuilder( // gather the H5typedef found in DataObjects if (nested.isTypedef) { val mdt = nested.dataObject!!.mdt!! + // if its a typedef but not a variable, promote to shared + if (!nested.isVariable) mdt.isShared = true val typename = if (mdt.isShared) nested.dataObject!!.name else null val typedef = H5typedef(typename, mdt) - typedefs.add(typedef) } } diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5reader.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5reader.kt index 063a3e40..94ea46e4 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5reader.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5reader.kt @@ -93,7 +93,7 @@ internal fun H5builder.processDataIntoArray(ba: ByteArray, isBE: Boolean, da } val tba = TypedByteArray(datatype, ba, 0, isBE = isBE) - return tba.convertToArrayTyped(shape) + return tba.convertToArrayTyped(shape, charToString = true) } // Put the variable length members (vlen, string) on the heap diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt index a5ded741..ae8b367d 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt @@ -4,6 +4,7 @@ package com.sunya.netchdf.hdf5 import com.sunya.cdm.api.* import com.sunya.cdm.api.Datatype.Companion.STRING +import com.sunya.cdm.api.Datatype.Companion.CHAR import com.sunya.cdm.array.ArrayEmpty import com.sunya.cdm.array.ArraySingle import com.sunya.cdm.array.ArrayString @@ -43,7 +44,11 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { val vinfo = v2.spObject as DataContainerVariable if (vinfo.onlyFillValue) { // fill value only, no data - if (v2.datatype == STRING) return ArrayString(intArrayOf(1), listOf("")) as ArrayTyped + if (v2.datatype == STRING) return ArrayString(v2.shape.toIntArray(), List(v2.nelems.toInt()) {""} ) as ArrayTyped + if (v2.datatype == CHAR) { + val shapeMinus1 = if (v2.rank == 0) intArrayOf(1) else IntArray(v2.rank - 1) { v2.shape[it].toInt() } + return ArrayString(shapeMinus1, List(shapeMinus1.computeSize()) {""} ) as ArrayTyped + } val tba = TypedByteArray(v2.datatype, vinfo.fillValue, 0, isBE = vinfo.h5type.isBE) return ArraySingle(wantSection.shape.toIntArray(), v2.datatype, tba.get(0)) } diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt index 5211154c..e2b570e5 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/TestFiles.kt @@ -4,7 +4,7 @@ import okio.FileSystem import okio.Path import okio.Path.Companion.toPath -const val testData = "core/src/commonTest/data/" +const val testData = "src/commonTest/data/" fun testFilesIn(dirPath: String): TestFiles.SequenceBuilder { return TestFiles.SequenceBuilder(dirPath) diff --git a/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt b/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt index 6a5b3ff9..50faafd1 100644 --- a/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt +++ b/testclibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt @@ -806,7 +806,7 @@ internal fun processDataIntoArray(ba: ByteArray, isBE: Boolean, datatype5 : // convert to array of Strings by reducing rank by 1, tricky shape shifting for non-scalars if (datatype5 == Datatype5.String) { - val extshape = IntArray(shape.size + 1) { if (it == shape.size) elemSize else shape[it] } + val extshape = if (elemSize == 1) shape else IntArray(shape.size + 1) { if (it == shape.size) elemSize else shape[it] } val result = ArrayUByte.fromByteArray(extshape, ba) return result.makeStringsFromBytes() as ArrayTyped } @@ -816,7 +816,7 @@ internal fun processDataIntoArray(ba: ByteArray, isBE: Boolean, datatype5 : } val tba = TypedByteArray(datatype, ba, 0, isBE = isBE) - return tba.convertToArrayTyped(shape) + return tba.convertToArrayTyped(shape, charToString = true) } // Put the variable length members (vlen, string) on the heap diff --git a/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/NClibFile.kt b/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/NClibFile.kt index 69d6c064..df6f149c 100644 --- a/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/NClibFile.kt +++ b/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/NClibFile.kt @@ -2,7 +2,6 @@ package com.sunya.netchdf.netcdfClib -import com.fleeksoft.charset.Platform import com.sunya.cdm.api.* import com.sunya.cdm.array.* import com.sunya.cdm.layout.MaxChunker @@ -13,7 +12,6 @@ import com.sunya.netchdf.netcdfClib.ffm.nc_vlen_t import com.sunya.netchdf.netcdfClib.ffm.netcdf_h.* import java.lang.foreign.* import java.lang.foreign.ValueLayout.* -import java.nio.ByteBuffer /* 1. Run /home/stormy/dev/github/netcdf/netcdf-c/rebuild.sh: @@ -138,7 +136,7 @@ class NClibFile(val filename: String) : Netchdf { } val shape = wantSection.shape.toIntArray() - when (datatype) { + return when (datatype) { Datatype.VLEN -> { val basetype = header.convertType(userType!!.baseTypeid) // an array of vlen structs. each vlen has an address and a size @@ -155,7 +153,7 @@ class NClibFile(val filename: String) : Netchdf { //println(" data=${adata.contentToString()}") listOfVlen.add( adata) } - return ArrayVlen.fromArray(shape, listOfVlen, basetype) as ArrayTyped + ArrayVlen.fromArray(shape, listOfVlen, basetype) as ArrayTyped // TODO nc_free_vlen(nc_vlen_t *vl); // nc_free_string(size_t len, char **data); // nc_reclaim_data() @@ -210,7 +208,7 @@ class NClibFile(val filename: String) : Netchdf { } ArrayVlen.fromArray(member.shape, listOfVlen, member.datatype.typedef!!.baseType) } - return sdataArray as ArrayTyped + sdataArray as ArrayTyped } Datatype.ENUM1, Datatype.ENUM2, Datatype.ENUM4 -> { @@ -223,15 +221,7 @@ class NClibFile(val filename: String) : Netchdf { val ba = val_p.toArray(ValueLayout.JAVA_BYTE)!! // TODO im skeptical isBE shouldnt always be nativeByteOrder val tba = TypedByteArray(v2.datatype, ba, 0, isBE = true) - return tba.convertToArrayTyped(shape) - /* - val values = ByteBuffer.wrap(raw) - when (datatype) { - Datatype.ENUM1 -> return ArrayUByte(shape, datatype as Datatype, values) as ArrayTyped - Datatype.ENUM2 -> return ArrayUShort(shape, datatype as Datatype, values) as ArrayTyped - Datatype.ENUM4 -> return ArrayUInt(shape, datatype as Datatype, values) as ArrayTyped - else -> throw RuntimeException() - } */ + tba.convertToArrayTyped(shape) } Datatype.BYTE -> { @@ -239,7 +229,7 @@ class NClibFile(val filename: String) : Netchdf { checkErr("nc_get_vars_schar", nc_get_vars_schar(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val raw = val_p.toArray(ValueLayout.JAVA_BYTE)!! - return ArrayByte(shape, raw) as ArrayTyped + ArrayByte(shape, raw) as ArrayTyped } Datatype.UBYTE -> { @@ -247,15 +237,28 @@ class NClibFile(val filename: String) : Netchdf { checkErr("nc_get_vars_uchar", nc_get_vars_uchar(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val raw = val_p.toArray(ValueLayout.JAVA_BYTE)!! - return ArrayUByte.fromByteArray(shape, raw) as ArrayTyped + ArrayUByte.fromByteArray(shape, raw) as ArrayTyped } Datatype.CHAR -> { val val_p = session.allocate(nelems) - checkErr("nc_get_vars_text", - nc_get_vars_text(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) + checkErr( + "nc_get_vars_text", + nc_get_vars_text(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p) + ) val raw = val_p.toArray(ValueLayout.JAVA_BYTE) - return ArrayUByte.fromByteArray(shape, Datatype.CHAR, raw) as ArrayTyped + val ubytea = ArrayUByte.fromByteArray(shape, Datatype.CHAR, raw) + + // * Netcdf-3 does not have STRING or UBYTE types, and in practice, CHAR is used for either. Variables of type CHAR + // return data as ArrayUByte. + // * Netcdf-4 encodes CHAR values as HDF5 string type with elemSize = 1, so we use that convention to detect + // legacy CHAR variables in HDF5 format. + // Variables of type CHAR return data as STRING, since users can use UBYTE if thats what they intend. + if (this.header.formatType.isNetdf3format) { + ubytea as ArrayTyped + } else { + ubytea.makeStringsFromBytes() as ArrayTyped// netcdf4 + } } Datatype.DOUBLE -> { @@ -263,14 +266,14 @@ class NClibFile(val filename: String) : Netchdf { val val_p = session.allocateArray(C_DOUBLE, nelems) checkErr("nc_get_vars_double", nc_get_vars_double(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_DOUBLE)!! - return ArrayDouble(shape, values) as ArrayTyped + ArrayDouble(shape, values) as ArrayTyped } Datatype.FLOAT -> { val val_p = session.allocateArray(C_FLOAT, nelems) checkErr("nc_get_vars_float", nc_get_vars_float(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_FLOAT)!! - return ArrayFloat(shape, values) as ArrayTyped + ArrayFloat(shape, values) as ArrayTyped } Datatype.INT -> { @@ -278,7 +281,7 @@ class NClibFile(val filename: String) : Netchdf { val val_p = session.allocateArray(C_INT, nelems) checkErr("nc_get_vars_int", nc_get_vars_int(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_INT)!! - return ArrayInt(shape, values) as ArrayTyped + ArrayInt(shape, values) as ArrayTyped } Datatype.UINT -> { @@ -286,7 +289,7 @@ class NClibFile(val filename: String) : Netchdf { val val_p = session.allocateArray(C_INT, nelems) checkErr("nc_get_vars_uint", nc_get_vars_uint(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_INT)!! - return ArrayUInt.fromIntArray(shape, values) as ArrayTyped + ArrayUInt.fromIntArray(shape, values) as ArrayTyped } Datatype.LONG -> { @@ -295,7 +298,7 @@ class NClibFile(val filename: String) : Netchdf { checkErr("nc_get_vars_long", nc_get_vars_long(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_LONG)!! - return ArrayLong(shape, values) as ArrayTyped + ArrayLong(shape, values) as ArrayTyped } Datatype.ULONG -> { @@ -303,20 +306,14 @@ class NClibFile(val filename: String) : Netchdf { val val_p = session.allocateArray(C_LONG as MemoryLayout, nelems) checkErr("nc_get_vars_ulonglong", nc_get_vars_ulonglong(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_LONG)!! - /* - val values = ByteBuffer.allocate(8 * nelems.toInt()) - val lvalues = values.asLongBuffer() - for (i in 0 until nelems) { - lvalues.put(i.toInt(), val_p.getAtIndex(C_LONG, i)) - } */ - return ArrayULong.fromLongArray(shape, values) as ArrayTyped + ArrayULong.fromLongArray(shape, values) as ArrayTyped } Datatype.SHORT -> { val val_p = session.allocateArray(C_SHORT, nelems) checkErr("nc_get_vars_short", nc_get_vars_short(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_SHORT)!! - return ArrayShort(shape, values) as ArrayTyped + ArrayShort(shape, values) as ArrayTyped } Datatype.USHORT -> { @@ -324,7 +321,7 @@ class NClibFile(val filename: String) : Netchdf { checkErr("nc_get_vars_ushort", nc_get_vars_ushort(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val values = val_p.toArray(C_SHORT)!! - return ArrayUShort.fromShortArray(shape, values) as ArrayTyped + ArrayUShort.fromShortArray(shape, values) as ArrayTyped } Datatype.STRING -> { @@ -336,14 +333,14 @@ class NClibFile(val filename: String) : Netchdf { val cString = address.reinterpret(Long.MAX_VALUE) values.add(cString.getUtf8String(0)) } - return ArrayString(shape, values) as ArrayTyped + ArrayString(shape, values) as ArrayTyped } Datatype.OPAQUE -> { val val_p = session.allocate(nelems * userType!!.size) checkErr("opaque nc_get_var", nc_get_vars(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p)) val ba = val_p.toArray(ValueLayout.JAVA_BYTE)!! - return ArrayOpaque.fromByteArray(shape, ba, userType.size) as ArrayTyped + ArrayOpaque.fromByteArray(shape, ba, userType.size) as ArrayTyped } else -> throw IllegalArgumentException("unsupported datatype ${datatype}") diff --git a/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/UserTypes.kt b/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/UserTypes.kt index 728570c3..1f2d2a91 100644 --- a/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/UserTypes.kt +++ b/testclibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/UserTypes.kt @@ -18,7 +18,7 @@ import java.lang.foreign.ValueLayout.JAVA_BYTE val debugUserTypes = false -// val nvars_p = session.allocate(C_INT, 0) +// val nvars_p = session.allocate(C_INT, 0) // checkErr("nc_inq_nvars", nc_inq_nvars(g4.grpid, nvars_p)) // val nvars = nvars_p[C_INT, 0] // @@ -232,18 +232,7 @@ private fun NCheader.readEnumAttValues(session: Arena, grpid: Int, varid: In // TODO im skeptical isBE shouldnt always be nativeByteOrder val tba = TypedByteArray(userType.enumBasePrimitiveType, ba, 0, isBE = true) val result = tba.convertToArrayTyped(intArrayOf(nelems.toInt())) - /* - val bb = ByteBuffer.wrap(raw) - val result = mutableListOf() - for (i in 0 until nelems.toInt()) { - val num = when (userType.enumBasePrimitiveType) { - Datatype.UBYTE -> bb.get(i).toUByte() - Datatype.USHORT -> bb.getShort(i).toUShort() - Datatype.UINT -> bb.getInt(i).toUInt() - else -> throw RuntimeException("convertEnums unknown type = ${userType.enumBasePrimitiveType}") - } - result.add(num) - } */ + attb.setValues(result.toList()) return attb } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt new file mode 100644 index 00000000..04d3ede6 --- /dev/null +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/CompareCdmWithClib.kt @@ -0,0 +1,158 @@ +package com.sunya.netchdf + +import com.sunya.cdm.api.Attribute +import com.sunya.cdm.api.CompoundTypedef +import com.sunya.cdm.api.Datatype +import com.sunya.cdm.api.EnumTypedef +import com.sunya.cdm.api.Group +import com.sunya.cdm.api.Netchdf +import com.sunya.cdm.api.OpaqueTypedef +import com.sunya.cdm.api.Typedef +import com.sunya.cdm.api.Variable +import com.sunya.cdm.api.VlenTypedef +import com.sunya.cdm.util.Indent +import com.sunya.netchdf.hdf4Clib.Hdf4ClibFile +import com.sunya.netchdf.hdf5Clib.Hdf5ClibFile +import com.sunya.netchdf.netcdfClib.NClibFile +import org.junit.jupiter.api.assertNotNull +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.test.fail +import kotlin.use + +class CompareCdmWithClib(filename: String, showCdl: Boolean = false, showCompare: Boolean = false) { + init { + println("=============================================================") + openNetchdfFile(filename).use { netchdf -> + if (netchdf == null) { + println("*** not a netchdf file = $filename") + } else { + println("${netchdf.type()} $filename ${"%.2f".format(netchdf.size / 1000.0 / 1000.0)} Mbytes") + if (showCdl) println("\nnetchdf ${netchdf.cdl()}") + + if (netchdf.type().contains("hdf4") || netchdf.type().contains("hdf-eos2")) { + Hdf4ClibFile(filename).use { ncfile -> + if (showCdl) println("\nHdf4ClibFile ${ncfile.cdl()}") + compareNetchdfCdm(netchdf, ncfile, showCompare = showCompare) + } + } else if (netchdf.type().contains("netcdf")) { + NClibFile(filename).use { ncfile -> + if (showCdl) println("\nNClibFile ${ncfile.cdl()}") + compareNetchdfCdm(netchdf, ncfile, showCompare = showCompare) + } + } else if (netchdf.type().contains("hdf5") || netchdf.type().contains("hdf-eos5")) { + Hdf5ClibFile(filename).use { ncfile -> + if (showCdl) println("\nHdf5ClibFile ${ncfile.cdl()}") + compareNetchdfCdm(netchdf, ncfile, showCompare = showCompare) + } + } else { + println("*** no c library to compare for $filename") + } + } + } + } +} + +fun compareNetchdfCdm(myfile: Netchdf, cfile: Netchdf, indent: Indent = Indent(2, startingLevel = 0), showCompare: Boolean = false) { + compareGroup(myfile.rootGroup(), cfile.rootGroup(), indent, showCompare) +} + +fun compareGroup(group: Group, cgroup: Group, indent: Indent, showCompare: Boolean) { + if (showCompare) { + println("${indent}group $group") + println("${indent}cgroup $cgroup") + } + + group.attributes.forEach { att -> + val catt = cgroup.findAttribute(att.name)!! + compareAttribute(att, catt, indent.incr(), showCompare) + } + + group.variables.forEach { v -> + val cv = cgroup.variables.find { it.name == v.name }!! + compareVariable(v, cv, indent.incr(), showCompare) + } + + group.groups.forEach { nested -> + val cnested = cgroup.findNestedGroupByShortName(nested.name)!! + compareGroup(nested, cnested, indent.incr(), showCompare) + } +} + +fun compareAttribute(att: Attribute<*>, catt: Attribute<*>, indent: Indent, showCompare: Boolean) { + if (showCompare) { + println("${indent}att $att") + println("${indent}catt $catt") + } + assertEquals(att.name, catt.name) + if (att.datatype == Datatype.OPAQUE) { + att.values.forEachIndexed { idx, it -> + val bval = it as ByteArray + val cval = catt.values[idx] as ByteArray + assertTrue(bval.contentEquals(cval)) + } + } else { + assertEquals(att.values, catt.values) + } +} + +fun compareVariable(v: Variable<*>, cv: Variable<*>, indent: Indent, showCompare: Boolean) { + if (showCompare) { + println("${indent}v ${v.nameAndShape()}") + println("${indent}cv ${cv.nameAndShape()}") + } + assertEquals(v.name, cv.name) + assertEquals(v.dimensions, cv.dimensions) + + v.attributes.forEach { att -> + val catt = cv.findAttribute(att.name)!! + compareAttribute(att, catt, indent.incr(), showCompare) + } + + compareDatatype(v.datatype, cv.datatype, indent.incr(), showCompare) +} + +fun compareDatatype(datatype: Datatype<*>, cdatatype: Datatype<*>, indent: Indent, showCompare: Boolean) { + if (showCompare) { + println("${indent}datatype ${datatype}") + println("${indent}cdatatype ${cdatatype}") + } + assertEquals (datatype.cdlName, cdatatype.cdlName) + compareTypedef(datatype.typedef, cdatatype.typedef, indent, showCompare) +} + +fun compareTypedef(t: Typedef?, ct: Typedef?, indent: Indent, showCompare: Boolean) { + if ((t == null) && (ct == null)) return + if ((t == null) && (ct != null)) { + fail("typedef missing; ctypedef = ${ct}") + } + if ((t != null) && (ct == null)) { + fail("ctypedef missing; typedef = ${t}") + } + assertNotNull(t) + assertNotNull(ct) + + if (showCompare) { + println("${t.cdl(indent)}") + println("${ct.cdl(indent)}") + } + + assertEquals(t.kind, ct.kind) + assertEquals(t.baseType, ct.baseType) + + if (t is OpaqueTypedef) { + assertTrue(ct is OpaqueTypedef) + assertEquals(t.elemSize, ct.elemSize) + + } else if (t is VlenTypedef) { + assertTrue(ct is VlenTypedef) + + } else if (t is EnumTypedef) { + assertTrue(ct is EnumTypedef) + assertEquals(t.valueMap, ct.valueMap) + + } else if (t is CompoundTypedef) { + assertTrue(ct is CompoundTypedef) + assertEquals(t.members, ct.members) + } +} \ No newline at end of file diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/CompareNetchdf.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/CompareNetchdf.kt deleted file mode 100644 index df69a4d4..00000000 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/CompareNetchdf.kt +++ /dev/null @@ -1,158 +0,0 @@ -package com.sunya.netchdf - -import com.sunya.cdm.api.Attribute -import com.sunya.cdm.api.CompoundTypedef -import com.sunya.cdm.api.Datatype -import com.sunya.cdm.api.EnumTypedef -import com.sunya.cdm.api.Group -import com.sunya.cdm.api.Netchdf -import com.sunya.cdm.api.OpaqueTypedef -import com.sunya.cdm.api.Typedef -import com.sunya.cdm.api.Variable -import com.sunya.cdm.api.VlenTypedef -import com.sunya.cdm.util.Indent -import com.sunya.netchdf.hdf4Clib.Hdf4ClibFile -import com.sunya.netchdf.hdf5Clib.Hdf5ClibFile -import com.sunya.netchdf.netcdfClib.NClibFile -import org.junit.jupiter.api.assertNotNull -import kotlin.test.assertEquals -import kotlin.test.assertTrue -import kotlin.test.fail -import kotlin.use - -class CompareNetchdf(filename: String, showCdl: Boolean = false, val showCompare: Boolean = false) { - init { - println("=============================================================") - openNetchdfFile(filename).use { netchdf -> - if (netchdf == null) { - println("*** not a netchdf file = $filename") - } else { - println("${netchdf.type()} $filename ${"%.2f".format(netchdf.size / 1000.0 / 1000.0)} Mbytes") - if (showCdl) println("\nnetchdf ${netchdf.cdl()}") - - if (netchdf.type().contains("hdf4") || netchdf.type().contains("hdf-eos2")) { - Hdf4ClibFile(filename).use { ncfile -> - if (showCdl) println("\nHdf4ClibFile ${ncfile.cdl()}") - compareNetchdf(netchdf, ncfile, Indent(2, startingLevel=0)) - } - } else if (netchdf.type().contains("netcdf")) { - NClibFile(filename).use { ncfile -> - if (showCdl) println("\nNClibFile ${ncfile.cdl()}") - compareNetchdf(netchdf, ncfile, Indent(2, startingLevel=0)) - } - } else if (netchdf.type().contains("hdf5") || netchdf.type().contains("hdf-eos5")) { - Hdf5ClibFile(filename).use { ncfile -> - if (showCdl) println("\nHdf5ClibFile ${ncfile.cdl()}") - compareNetchdf(netchdf, ncfile, Indent(2, startingLevel=0)) - } - } else { - println("*** no c library to compare for $filename") - } - } - } - } - - fun compareNetchdf(myfile: Netchdf, cfile: Netchdf, indent: Indent) { - compareGroup(myfile.rootGroup(), cfile.rootGroup(), indent) - } - - fun compareGroup(group: Group, cgroup: Group, indent: Indent) { - if (showCompare) { - println("${indent}group $group") - println("${indent}cgroup $cgroup") - } - - group.attributes.forEach { att -> - val catt = cgroup.findAttribute(att.name)!! - compareAttribute(att, catt, indent.incr()) - } - - group.variables.forEach { v -> - val cv = cgroup.variables.find { it.name == v.name }!! - compareVariable(v, cv, indent.incr()) - } - - group.groups.forEach { nested -> - val cnested = cgroup.findNestedGroupByShortName(nested.name)!! - compareGroup(nested, cnested, indent.incr()) - } - } - - fun compareAttribute(att: Attribute<*>, catt: Attribute<*>, indent: Indent) { - if (showCompare) { - println("${indent}att $att") - println("${indent}catt $catt") - } - assertEquals(att.name, catt.name) - if (att.datatype == Datatype.OPAQUE) { - att.values.forEachIndexed { idx, it -> - val bval = it as ByteArray - val cval = catt.values[idx] as ByteArray - assertTrue(bval.contentEquals(cval)) - } - } else { - assertEquals(att.values, catt.values) - } - } - - fun compareVariable(v: Variable<*>, cv: Variable<*>, indent: Indent) { - if (showCompare) { - println("${indent}v ${v.nameAndShape()}") - println("${indent}cv ${cv.nameAndShape()}") - } - assertEquals(v.name, cv.name) - assertEquals(v.dimensions, cv.dimensions) - - v.attributes.forEach { att -> - val catt = cv.findAttribute(att.name)!! - compareAttribute(att, catt, indent.incr()) - } - - compareDatatype(v.datatype, cv.datatype, indent.incr()) - } - - fun compareDatatype(datatype: Datatype<*>, cdatatype: Datatype<*>, indent: Indent) { - if (showCompare) { - println("${indent}datatype ${datatype}") - println("${indent}cdatatype ${cdatatype}") - } - assertEquals (datatype.cdlName, cdatatype.cdlName) - compareTypedef(datatype.typedef, cdatatype.typedef, indent) - } - - fun compareTypedef(t: Typedef?, ct: Typedef?, indent: Indent) { - if ((t == null) && (ct == null)) return - if ((t == null) && (ct != null)) { - fail("typedef missing; ctypedef = ${ct}") - } - if ((t != null) && (ct == null)) { - fail("ctypedef missing; typedef = ${t}") - } - assertNotNull(t) - assertNotNull(ct) - - if (showCompare) { - println("${t.cdl(indent)}") - println("${ct.cdl(indent)}") - } - - assertEquals(t.kind, ct.kind) - assertEquals(t.baseType, ct.baseType) - - if (t is OpaqueTypedef) { - assertTrue(ct is OpaqueTypedef) - assertEquals(t.elemSize, ct.elemSize) - - } else if (t is VlenTypedef) { - assertTrue(ct is VlenTypedef) - - } else if (t is EnumTypedef) { - assertTrue(ct is EnumTypedef) - assertEquals(t.valueMap, ct.valueMap) - - } else if (t is CompoundTypedef) { - assertTrue(ct is CompoundTypedef) - assertEquals(t.members, ct.members) - } - } -} \ No newline at end of file diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt index 7c1102ff..159a388a 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/KnownProblemFiles.kt @@ -71,7 +71,7 @@ group: the_out_crowd { } */ @Test fun tst_grps() { - CompareNetchdf(testData + "devcdm/netcdf4/tst_grps.nc4") + CompareCdmWithClib(testData + "devcdm/netcdf4/tst_grps.nc4") } @Test @@ -84,7 +84,7 @@ group: the_out_crowd { // float field3 ; // }; // compound_att_float fun compoundAttributeTest() { - CompareNetchdf(testData + "cdmUnitTest/formats/netcdf4/compound-attribute-test.nc") + CompareCdmWithClib(testData + "cdmUnitTest/formats/netcdf4/compound-attribute-test.nc") } // We use HDFEOS_INFORMATION to modify the structure, Nclib does not. @@ -229,11 +229,11 @@ group: the_out_crowd { //> but was: - CompareNetchdf(filename, showCdl = true) + CompareCdmWithClib(filename, showCdl = false) } } @Test - fun readNetchdfData() { + fun testCompareDataWithClib() { files().forEach { filename -> - readNetchdfData(filename, null) + compareDataWithClib(filename) } } @Test - fun testCompareDataWithClib() { + fun testFilesAfter() { + var skip = true files().forEach { filename -> - compareDataWithClib(filename) + if (filename.equals(testData + "netchdf/martaan/RADNL_TEST_R___25PCPRR_L3__20090305T120000_20090305T120500_0001.nc")) skip = false + if (!skip) compareDataWithClib(filename) } } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt index cc8e1014..4f2c8ade 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibTest.kt @@ -54,16 +54,16 @@ class NetchdfClibTest { @Test fun testOneCdl() { val filename = testData + "netchdf/tomas/S3A_OL_CCDB_CHAR_AllFiles.20101019121929_1.nc4" - CompareNetchdf(filename) + CompareCdmWithClib(filename) } @Test fun testEnums() { - CompareNetchdf(testData + "devcdm/netcdf4/test_enum_type.nc") - CompareNetchdf(testData + "devcdm/netcdf4/tst_enums.nc") - CompareNetchdf(testData + "devcdm/hdf5/enumcmpnd.h5") - CompareNetchdf(testData + "devcdm/hdf5/enum.h5") - CompareNetchdf(testData + "devcdm/hdf5/cenum.h5") + CompareCdmWithClib(testData + "devcdm/netcdf4/test_enum_type.nc") + CompareCdmWithClib(testData + "devcdm/netcdf4/tst_enums.nc") + CompareCdmWithClib(testData + "devcdm/hdf5/enumcmpnd.h5") + CompareCdmWithClib(testData + "devcdm/hdf5/enum.h5") + CompareCdmWithClib(testData + "devcdm/hdf5/cenum.h5") compareDataWithClib(testData + "devcdm/netcdf4/test_enum_type.nc") compareDataWithClib(testData + "devcdm/netcdf4/tst_enums.nc") @@ -86,7 +86,7 @@ class NetchdfClibTest { fun problem2() { val filename = testData + "devcdm/hdf5/enumcmpnd.h5" // compareN4withH5cdl(filename) - CompareNetchdf(filename) + CompareCdmWithClib(filename) compareDataWithClib(filename) } @@ -199,7 +199,7 @@ isThreadsafe = 0 = false @Test fun testHdf4Attribute() { val filename = testData + "/hdf4/eos/misr/MISR_AM1_GRP_TERR_GM_P040_AN" - CompareNetchdf(filename) + CompareCdmWithClib(filename) compareDataWithClib(filename, ) } @@ -213,14 +213,14 @@ isThreadsafe = 0 = false @Test fun testFailDataCompare3() { val filename = testData + "/devcdm/hdfeos2/MISR_AM1_GP_GMP_P040_O003734_05.eos" - CompareNetchdf(filename, true) + CompareCdmWithClib(filename, true) compareDataWithClib(filename, "/GeometricParameters/Data_Fields/SolarAzimuth") } @Test fun testFailDataCompare4() { val filename = testData + "/devcdm/netcdf4/tst_opaque_data.nc4" - CompareNetchdf(filename, true) + CompareCdmWithClib(filename, true) compareDataWithClib(filename) } @@ -252,23 +252,28 @@ isThreadsafe = 0 = false @Test fun testVlenAttribute() { val filename = "/home/all/testdata/devcdm/netcdf4/tst_solar_2.nc4" - CompareNetchdf(filename, true, true) + CompareCdmWithClib(filename, true, true) } - // TODO a message with only mdt but says isShared = false @Test fun problemCompare() { val filename = "/home/all/testdata/devcdm/hdf4/MAC07S0.A2008230.1250.002.2008233222357.hdf" - CompareNetchdf(filename, true, true) + CompareCdmWithClib(filename, true, true) + } + + @Test + fun problem() { + val filename = "/home/all/testdata/netchdf/austin/H12007_1m_MLLW_1of6.bag" + CompareCdmWithClib(filename, true, true) } @Test fun compareNetchdf() { files().forEach { filename -> try { - CompareNetchdf(filename, false, false) + CompareCdmWithClib(filename, false, false) } catch (e: Throwable) { - CompareNetchdf(filename, true, true) + CompareCdmWithClib(filename, true, true) e.printStackTrace() } } @@ -277,7 +282,7 @@ isThreadsafe = 0 = false @Test fun testCdlWithClib() { files().forEach { filename -> - CompareNetchdf(filename) + CompareCdmWithClib(filename) } } @@ -295,6 +300,15 @@ isThreadsafe = 0 = false } } + // @Test + fun testFilesAfter() { + var skip = true + NetchdfClibExtra.Companion.files().forEach { filename -> + if (filename.equals(testData + "netchdf/martaan/RADNL_TEST_R___25PCPRR_L3__20090305T120000_20090305T120500_0001.nc")) skip = false + if (!skip) compareDataWithClib(filename) + } + } + //@Test //@MethodSource("params") fun testIterateWithClib() { @@ -343,7 +357,7 @@ fun readNcData(filename: String, varname: String? = null, section: SectionPartia } } -fun compareDataWithClib(filename: String, varname: String? = null, section: SectionPartial? = null) { +fun compareDataWithClib(filename: String, varname: String? = null, section: SectionPartial? = null, showCdl: Boolean = false) { println("=============================================================") openNetchdfFile(filename).use { netchdf -> if (netchdf == null) { @@ -351,7 +365,7 @@ fun compareDataWithClib(filename: String, varname: String? = null, section: Sect return } println("${netchdf.type()} $filename ${"%.2f".format(netchdf.size / 1000.0 / 1000.0)} Mbytes") - if (NetchdfClibTest.showCdl) println("\n${netchdf.cdl()}") + if (showCdl) println("\n${netchdf.cdl()}") if (netchdf.type().contains("hdf4") || netchdf.type().contains("hdf-eos2")) { Hdf4ClibFile(filename).use { ncfile -> diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/TestNPP.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/TestNPP.kt index b11efac4..0eecca93 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/TestNPP.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/TestNPP.kt @@ -175,7 +175,7 @@ What we get though are just the last part of the names, not a full path: @Test fun problemNPP() { val filename = testData + "netchdf/npp/VCBHO_npp_d20030125_t084955_e085121_b00015_c20071213022754_den_OPS_SEG.h5" - CompareNetchdf(filename, showCdl = true) + CompareCdmWithClib(filename, showCdl = true) readNetchdfData(filename, ) compareDataWithClib(filename, ) } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt index cee3eb7a..03184759 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt @@ -225,14 +225,6 @@ class H4Ccompare { } } - // @Test - fun readH4dataAll() { - files().forEach { filename -> - readNetchdfData(filename, null, null, true) - println() - } - } - @Test fun compareDataAll() { files().forEach { filename -> diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt index c0a56995..45c704de 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt @@ -3,6 +3,7 @@ package com.sunya.netchdf.hdf5 import com.sunya.cdm.api.Datatype import com.sunya.netchdf.* import com.sunya.netchdf.netcdfClib.NClibFile +import com.sunya.netchdf.testfiles.H5Files import com.sunya.netchdf.testfiles.N4Files import com.sunya.netchdf.testutils.testData import kotlin.test.* @@ -18,47 +19,35 @@ class Hdf5Compare { fun files(): Iterator { // 10 of 114 fail, because we compare with netcdf4 instead of hdf5 c library - //val hdfeos5 = - // testFilesIn(testData + "devcdm/hdfeos5") - // .withRecursion() - // .build() - - return N4Files.files() - // return Stream.of( N4Files.params(), H5Files.params()).flatMap { i -> i }; + return H5Files.files() } } @Test fun testNewLibrary() { val filename = testData + "netchdf/haberman/iso.h5" - CompareNetchdf(filename, showCdl = true) + CompareCdmWithClib(filename, showCdl = true) compareDataWithClib(filename) } @Test fun problemChars() { val filename = testData + "cdmUnitTest/formats/netcdf4/files/c0_4.nc4" - CompareNetchdf(filename) - compareDataWithClib(filename) + CompareCdmWithClib(filename) + compareDataWithClib(filename, showCdl = true, varname = "c213") } @Test fun problemLibraryVersion() { val filename = testData + "devcdm/netcdf4/tst_solar_cmp.nc" - CompareNetchdf(filename, showCdl = true) + CompareCdmWithClib(filename, showCdl = true) compareDataWithClib(filename) } - @Test - fun ok() { - compareH5andNclib(testData + "netchdf/tomas/S3A_OL_CCDB_CHAR_AllFiles.20101019121929_1.nc4") - compareDataWithClib(testData + "netchdf/tomas/S3A_OL_CCDB_CHAR_AllFiles.20101019121929_1.nc4") - } - @Test fun problem() { - val filename = testData + "cdmUnitTest/formats/netcdf4/files/xma022032.nc" - CompareNetchdf(filename) + val filename = testData + "devcdm/netcdf4/IntTimSciSamp.nc" + CompareCdmWithClib(filename, showCdl = true) compareDataWithClib(filename) } @@ -78,9 +67,9 @@ class Hdf5Compare { } @Test - fun testCdlWithClib() { + fun compareNetchdf() { files().forEach { filename -> - CompareNetchdf(filename) + CompareCdmWithClib(filename) } } @@ -91,26 +80,6 @@ class Hdf5Compare { } } - @Test - fun compareH5andNclib() { - files().forEach { filename -> - compareH5andNclib(filename) - } - } - - fun compareH5andNclib(filename: String) { - println("===================================================") - openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF5).use { h5file -> - println("${h5file!!.type()} $filename ") - println("\n${h5file.cdl()}") - - NClibFile(filename).use { nclibfile -> - println("ncfile = ${nclibfile.cdl()}") - compareCdlWithoutFileType(nclibfile.cdl(), h5file.cdl()) - } - } - } - @Test fun readCharDataCompareNC() { files().forEach { filename -> diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5openTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5openTest.kt index 1a864026..6f7e54f2 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5openTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5openTest.kt @@ -84,25 +84,25 @@ class Hdf5openTest { @Test fun vlstra() { - CompareNetchdf(testData + "devcdm/hdf5/vlstra.h5") + CompareCdmWithClib(testData + "devcdm/hdf5/vlstra.h5") } //// the following wont work opening as netcdf @Test fun notNetcdf() { - CompareNetchdf(testData + "devcdm/hdf5/compound_complex.h5", showCdl = true) - CompareNetchdf(testData + "devcdm/hdf5/bitfield.h5", showCdl = true) - CompareNetchdf(testData + "devcdm/hdf5/SDS_array_type.h5", showCdl = true) + CompareCdmWithClib(testData + "devcdm/hdf5/compound_complex.h5", showCdl = true) + CompareCdmWithClib(testData + "devcdm/hdf5/bitfield.h5", showCdl = true) + CompareCdmWithClib(testData + "devcdm/hdf5/SDS_array_type.h5", showCdl = true) } @Test fun privateTypedef() { - CompareNetchdf(testData + "devcdm/hdf5/bitop.h5", showCdl = true) + CompareCdmWithClib(testData + "devcdm/hdf5/bitop.h5", showCdl = true) } @Test fun problemCompareCdl() { - CompareNetchdf(testData + "devcdm/netcdf4/testNestedStructure.nc", showCdl = true) + CompareCdmWithClib(testData + "devcdm/netcdf4/testNestedStructure.nc", showCdl = true) } /////////////////////////////////////////////////////////////////////////////////// @@ -130,7 +130,7 @@ class Hdf5openTest { @Test fun testCdlWithClibAll() { files().forEach { filename -> - CompareNetchdf(filename) + CompareCdmWithClib(filename) } } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt index 66e20696..61721a92 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/JhdfReadTest.kt @@ -1,6 +1,6 @@ package com.sunya.netchdf.hdf5 -import com.sunya.netchdf.CompareNetchdf +import com.sunya.netchdf.CompareCdmWithClib import com.sunya.netchdf.compareDataWithClib import com.sunya.netchdf.testfiles.JhdfFiles import kotlin.test.Test @@ -21,27 +21,27 @@ class JhdfReadTest { @Test fun problem() { - CompareNetchdf("../core/src/commonTest/data/jhdf/compound_datasets_earliest.hdf5", showCdl = true, showCompare = true) + CompareCdmWithClib("../core/src/commonTest/data/jhdf/compound_datasets_earliest.hdf5", showCdl = true, showCompare = true) // CompareNetchdf("../core/src/commonTest/data/jhdf/compound_datasets_earliest.hdf5", showCdl= true) } @Test fun problem2() { - CompareNetchdf("../core/src/commonTest/data/jhdf/compound_datasets_latest.hdf5", showCdl= true, showCompare = true) + CompareCdmWithClib("../core/src/commonTest/data/jhdf/compound_datasets_latest.hdf5", showCdl= true, showCompare = true) } @Test fun failure() { - CompareNetchdf("../core/src/commonTest/data/jhdf/globalheaps_test.hdf5", showCdl= true, showCompare = true) + CompareCdmWithClib("../core/src/commonTest/data/jhdf/globalheaps_test.hdf5", showCdl= true, showCompare = true) } @Test fun compareNetchdf() { files().forEach { filename -> try { - CompareNetchdf(filename, false, false) + CompareCdmWithClib(filename, false, false) } catch (e: Throwable) { - CompareNetchdf(filename, true, true) + CompareCdmWithClib(filename, true, true) e.printStackTrace() } } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt index 5946b889..2d633c0b 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/hdf5Clib/H5CopenTest.kt @@ -3,7 +3,6 @@ package com.sunya.netchdf.hdf5Clib import com.sunya.cdm.api.SectionPartial import com.sunya.netchdf.* import com.sunya.netchdf.testfiles.H5CFiles -import com.sunya.netchdf.testutils.Stats import com.sunya.netchdf.testutils.testData import kotlin.test.* @@ -16,23 +15,12 @@ class H5CopenTest { return H5CFiles.files() } - fun beforeAll() { - Stats.clear() - } - - fun afterAll() { - if (versions.size > 0) { - versions.keys.forEach { println("$it = ${versions[it]!!.size} files") } - } - Stats.show() - } - private val versions = mutableMapOf>() } @Test fun problemReferenceObjects() { - CompareNetchdf(testData + "cdmUnitTest/formats/hdf5/msg/test.h5") + CompareCdmWithClib(testData + "cdmUnitTest/formats/hdf5/msg/test.h5") } @Test @@ -40,12 +28,12 @@ class H5CopenTest { val filename = testData + "cdmUnitTest/formats/hdf5/aura/MLS-Aura_L2GP-BrO_v01-52-c01_2007d029.he5" //showNetchdfHeader(filename) //readNetchdfData(filename) - CompareNetchdf(filename) + CompareCdmWithClib(filename) } @Test fun dimScales() { - CompareNetchdf(testData + "cdmUnitTest/formats/hdf5/extLink/JA2_IPR_2PcP021_001_20090126_085038_20090126_094651.nc") + CompareCdmWithClib(testData + "cdmUnitTest/formats/hdf5/extLink/JA2_IPR_2PcP021_001_20090126_085038_20090126_094651.nc") } // netcdf cstr.h5 { @@ -60,7 +48,7 @@ class H5CopenTest { @Test fun fixlengthStringsInCompound() { val filename = testData + "devcdm/hdf5/cstr.h5" - CompareNetchdf(filename) + CompareCdmWithClib(filename) readNetchdfData(filename, "Compound_String") compareDataWithClib(filename) } @@ -96,7 +84,7 @@ class H5CopenTest { @Test fun compactStorage() { val filename = testData + "devcdm/hdf5/matlab_cols.mat" - CompareNetchdf(filename) + CompareCdmWithClib(filename) readNetchdfData(filename, "b") openH5C(filename, "b") compareDataWithClib(filename) @@ -105,7 +93,7 @@ class H5CopenTest { @Test fun problem() { val filename = testData + "devcdm/hdf5/SDS_array_type.h5" - CompareNetchdf(filename) + CompareCdmWithClib(filename) compareDataWithClib(filename, "IntArray") } @@ -114,7 +102,7 @@ class H5CopenTest { // but netcdf4 /home/all/testdata/cdmUnitTest/formats/hdf5/extLink/JA2_IPR_2PcP021_001_20090126_085038_20090126_094651.nc seems to work fun externalLinks() { val filename = testData + "formats/hdf5/extLink/extlink_source.h5" - CompareNetchdf(filename) + CompareCdmWithClib(filename) compareDataWithClib(filename, "IntArray") } @@ -153,7 +141,7 @@ class H5CopenTest { @Test fun testCdlWithH5Clib() { files().forEach { filename -> - CompareNetchdf(filename) + CompareCdmWithClib(filename) } } diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf3/N3dataCompare.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf3/N3dataCompare.kt index d95caf5d..3e56f486 100644 --- a/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf3/N3dataCompare.kt +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf3/N3dataCompare.kt @@ -16,26 +16,6 @@ class N3dataCompare { } } -/* - @Test - fun netcdf3() { - val filename = testData + "cdmUnitTest/formats/netcdf3/awips.nc" - if (showDetail) println("===============================================") - if (showDetail) { - Netcdf3File(filename).use { ncfile -> - println("${ncfile.type()} $filename ") - println("${ncfile.cdl()} ") - } - } - readData(filename, "uw", SectionPartial.fromSpec("4, 39, 55, 74")) - readData(filename, "uw", SectionPartial.fromSpec("0:4,13:26,18:37,25:49")) - readData(filename, "vw", SectionPartial.fromSpec("4, 39, 55, 74")) - readData(filename, "vw", SectionPartial.fromSpec("0:4,13:26,18:37,25:49")) - readData(filename, "uw", SectionPartial.fromSpec(":,:,:,25")) - } - - */ - @Test fun cdf5() { val filename = testData + "recent/cdf5/jays_DOMAIN000.nc" diff --git a/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt b/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt new file mode 100644 index 00000000..05f1bbc1 --- /dev/null +++ b/testclibs/src/test/kotlin/com/sunya/netchdf/netcdf4/CompareH5andNclib.kt @@ -0,0 +1,50 @@ +package com.sunya.netchdf.netcdf4 + +import com.sunya.netchdf.NetchdfFileFormat +import com.sunya.netchdf.compareDataWithClib +import com.sunya.netchdf.compareNetchdfCdm +import com.sunya.netchdf.netcdfClib.NClibFile +import com.sunya.netchdf.openNetchdfFileWithFormat +import com.sunya.netchdf.testfiles.N4Files +import com.sunya.netchdf.testutils.testData +import kotlin.test.Test +import kotlin.use + +class CompareH5andNclib { + + companion object { + @JvmStatic + fun files(): Iterator { + // 10 of 114 fail, because we compare with netcdf4 instead of hdf5 c library + + return N4Files.files() + } + } + + + @Test + fun problem() { + compareH5andNclib(testData + "netchdf/tomas/S3A_OL_CCDB_CHAR_AllFiles.20101019121929_1.nc4") + compareDataWithClib(testData + "netchdf/tomas/S3A_OL_CCDB_CHAR_AllFiles.20101019121929_1.nc4") + } + + @Test + fun compareH5andNclib() { + files().forEach { filename -> + compareH5andNclib(filename) + } + } + + fun compareH5andNclib(filename: String) { + println("===================================================") + openNetchdfFileWithFormat(filename, NetchdfFileFormat.HDF5).use { h5file -> + println("${h5file!!.type()} $filename ") + println("\n${h5file.cdl()}") + + NClibFile(filename).use { nclibfile -> + println("NClibFile = ${nclibfile.cdl()}") + compareNetchdfCdm(nclibfile, h5file) + } + } + } +} \ No newline at end of file