From eea9cd490bce98cb660558a680e753bb74c9653e Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 14 Aug 2026 12:57:05 +0800 Subject: [PATCH 1/4] Cover the streamed chunks in the catalog FITS and VOT test --- ICD_test_stages/catalog.tests | 1 + docs/source/catalog.rst | 33 ++++-- src/test/CATALOG_FITS_VOT.test.ts | 186 +++++++++++++++++++++++------- 3 files changed, 169 insertions(+), 51 deletions(-) diff --git a/ICD_test_stages/catalog.tests b/ICD_test_stages/catalog.tests index 1cef7c98..ef4012ed 100644 --- a/ICD_test_stages/catalog.tests +++ b/ICD_test_stages/catalog.tests @@ -1 +1,2 @@ src/test/CATALOG_GENERAL.test.ts +src/test/CATALOG_FITS_VOT.test.ts diff --git a/docs/source/catalog.rst b/docs/source/catalog.rst index a80f251c..701f2d06 100644 --- a/docs/source/catalog.rst +++ b/docs/source/catalog.rst @@ -217,7 +217,7 @@ CATALOG_FITS_VOT See the `source code `__. -This test verifies catalog operations for both FITS and VOTable catalog formats using a large catalog (COSMOSOPTCAT with 918,827 entries), comparing results between the two formats. +This test verifies catalog operations for both FITS and VOTable catalog formats using a large catalog (COSMOSOPTCAT with 918,827 entries), comparing results between the two formats. The same table is stored in both formats, so the two runs differ only in the file type and the file size, and every other part of the response has to agree. **For each catalog format (COSMOSOPTCAT.fits and COSMOSOPTCAT.vot):** @@ -269,8 +269,12 @@ This test verifies catalog operations for both FITS and VOTable catalog formats :red-text:`Check 3:` the CATALOG_FILE_INFO_RESPONSE should satisfy: - - COSMOSOPTCAT.fits: success = True, file_size = 444729600, description = "Count: 918827", headers length = 62 - - COSMOSOPTCAT.vot: success = True, file_size = 1631311089, description = "Count: 918827", headers length = 62 + - COSMOSOPTCAT.fits: success = True, file_info.type = FITSTable, file_size = 444729600, headers length = 62 + - COSMOSOPTCAT.vot: success = True, file_info.type = VOTable, file_size = 1631311089, headers length = 62 + + - file_info.description reports the file name, "Column Count: 62" and "Row Count: 918827" + + - the headers describe every column exactly once: their column_index values cover 0 to 61, each header has a non-empty name, and none has an UnsupportedType data type 7. Frontend sends: **OPEN_CATALOG_FILE** (``OpenCatalogFile``) @@ -286,6 +290,9 @@ This test verifies catalog operations for both FITS and VOTable catalog formats - Should arrive within 100000 ms - Both formats: success = True, data_size = 918827, headers length = 62 + - COSMOSOPTCAT.fits: file_id = 1, file_info.type = FITSTable; COSMOSOPTCAT.vot: file_id = 2, file_info.type = VOTable + + - preview_data has one entry per column (62), and every entry holds preview_data_size (50) rows 9. Frontend sends: **CATALOG_FILTER_REQUEST** (``CatalogFilterRequest``) to retrieve remaining rows @@ -296,13 +303,25 @@ This test verifies catalog operations for both FITS and VOTable catalog formats subset_data_size = 918777 subset_start_index = 50 -10. Backend returns: **CATALOG_FILTER_RESPONSE** (``CatalogFilterResponse``) +10. Backend streams: **CATALOG_FILTER_RESPONSE** (``CatalogFilterResponse``) -:red-text:`Check 5:` the CATALOG_FILTER_RESPONSE should satisfy: + The requested subset is not returned in one message. The backend splits it into chunks of at + most 100000 rows, so the 918777 requested rows arrive as 10 messages: nine of 100000 rows and a + final one of 18777 rows. + +:red-text:`Check 5:` the streamed CATALOG_FILTER_RESPONSE should satisfy: - Should arrive within 100000 ms - - Length of columns = 10, progress = 1 - - subsetDataSize = 18777, subsetEndIndex = 918827, filterDataSize = 918827 + + - Exactly 10 messages are streamed + + - Every message: file_id = 1 / 2, filter_data_size = 918827, request_end_index = 918827, and the columns are keyed by the requested column_indices [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + - The chunks cover the requested subset without a gap or an overlap: each message begins where the previous one ended, subset_end_index - subset_data_size gives its first row, and every column in it carries subset_data_size rows + + - The progress increases from message to message, stays below 1 until the last message, and equals 1 only in the last one + + - Last message: subset_data_size = 18777, subset_end_index = 918827 CATALOG_LARGE ~~~~~~~~~~~~~ diff --git a/src/test/CATALOG_FITS_VOT.test.ts b/src/test/CATALOG_FITS_VOT.test.ts index 0c3c930d..4ea3de69 100644 --- a/src/test/CATALOG_FITS_VOT.test.ts +++ b/src/test/CATALOG_FITS_VOT.test.ts @@ -12,15 +12,44 @@ let openCatalogLargeTimeout: number = config.timeout.openCatalogLarge; interface ICatalogFileInfoResponseExt extends CARTA.ICatalogFileInfoResponse { lengthOfHeaders: number; + descriptionKeywords: string[]; } interface IOpenCatalogFileAckExt extends CARTA.IOpenCatalogFileAck { lengthOfHeaders: number; + lengthOfPreviewData: number; } interface ICatalogFilterResponseExt extends CARTA.ICatalogFilterResponse { lengthOfColumns: number; - fileid: number; + // The backend streams the requested subset in chunks of TableController's max_chunk_size + maxChunkSize: number; + numberOfResponses: number; +} + +// ColumnData carries every type other than String as binary, so the number of rows in a +// column has to be derived from the byte length of its payload. +const bytesPerElement = new Map([ + [CARTA.ColumnType.Uint8, 1], + [CARTA.ColumnType.Int8, 1], + [CARTA.ColumnType.Bool, 1], + [CARTA.ColumnType.Uint16, 2], + [CARTA.ColumnType.Int16, 2], + [CARTA.ColumnType.Uint32, 4], + [CARTA.ColumnType.Int32, 4], + [CARTA.ColumnType.Float, 4], + [CARTA.ColumnType.Uint64, 8], + [CARTA.ColumnType.Int64, 8], + [CARTA.ColumnType.Double, 8], +]); + +function columnRowCount(column: CARTA.IColumnData): number { + if (column.dataType === CARTA.ColumnType.String) { + return column.stringData!.length; + } + const elementSize = bytesPerElement.get(column.dataType!)!; + expect(elementSize).toBeDefined(); + return column.binaryData!.length / elementSize; } interface AssertItem { @@ -107,29 +136,35 @@ let assertItem: AssertItem = { { fileInfo: { name: 'COSMOSOPTCAT.fits', + type: CARTA.CatalogFileType.FITSTable, fileSize: 444729600, - description: 'Count: 918827', }, success: true, lengthOfHeaders: 62, + descriptionKeywords: ['Name: COSMOSOPTCAT.fits', 'Column Count: 62', 'Row Count: 918827'], }, { fileInfo: { name: 'COSMOSOPTCAT.vot', - type: 1, + type: CARTA.CatalogFileType.VOTable, fileSize: 1631311089, - description: 'Count: 918827', }, success: true, lengthOfHeaders: 62, + descriptionKeywords: ['Name: COSMOSOPTCAT.vot', 'Column Count: 62', 'Row Count: 918827'], }, ], openCatalogFileAck: [ { dataSize: 918827, fileId: 1, - fileInfo: { name: 'COSMOSOPTCAT.fits', fileSize: 444729600 }, + fileInfo: { + name: 'COSMOSOPTCAT.fits', + type: CARTA.CatalogFileType.FITSTable, + fileSize: 444729600, + }, lengthOfHeaders: 62, + lengthOfPreviewData: 62, success: true, }, { @@ -137,10 +172,11 @@ let assertItem: AssertItem = { fileId: 2, fileInfo: { name: 'COSMOSOPTCAT.vot', - type: 1, + type: CARTA.CatalogFileType.VOTable, fileSize: 1631311089, }, lengthOfHeaders: 62, + lengthOfPreviewData: 62, success: true, }, ], @@ -171,7 +207,9 @@ let assertItem: AssertItem = { catalogFilterResponse: [ { lengthOfColumns: 10, - fileid: 1, + fileId: 1, + maxChunkSize: 100000, + numberOfResponses: 10, subsetDataSize: 18777, subsetEndIndex: 918827, filterDataSize: 918827, @@ -180,7 +218,9 @@ let assertItem: AssertItem = { }, { lengthOfColumns: 10, - fileid: 1, + fileId: 2, + maxChunkSize: 100000, + numberOfResponses: 10, subsetDataSize: 18777, subsetEndIndex: 918827, filterDataSize: 918827, @@ -264,30 +304,43 @@ assertItem.catalogFileInfoReq.map((data, index) => { ); }); + let CatalogFileInfoAck: CARTA.ICatalogFileInfoResponse; test(`(Step 4) Request CatalogFileInfo & check CatalogFileInfoAck | `, async () => { - let CatalogFileInfoAck = await msgController.getCatalogFileInfo( + CatalogFileInfoAck = await msgController.getCatalogFileInfo( assertItem.catalogFileInfoReq[index].directory, assertItem.catalogFileInfoReq[index].name ); expect(CatalogFileInfoAck.success).toEqual(assertItem.catalogFileInfoResponse[index].success); expect(CatalogFileInfoAck.fileInfo.name).toEqual(assertItem.catalogFileInfoResponse[index].fileInfo.name); - if (CatalogFileInfoAck.fileInfo.type) { - expect(CatalogFileInfoAck.fileInfo.type).toEqual( - assertItem.catalogFileInfoResponse[index].fileInfo.type - ); - } + // CatalogFileType.FITSTable is 0, so the type has to be compared unconditionally + expect(CatalogFileInfoAck.fileInfo.type).toEqual(assertItem.catalogFileInfoResponse[index].fileInfo.type); expect(CatalogFileInfoAck.fileInfo.fileSize.low).toEqual( assertItem.catalogFileInfoResponse[index].fileInfo.fileSize ); + assertItem.catalogFileInfoResponse[index].descriptionKeywords.forEach((keyword) => { + expect(CatalogFileInfoAck.fileInfo.description).toContain(keyword); + }); expect(CatalogFileInfoAck.headers.length).toEqual( assertItem.catalogFileInfoResponse[index].lengthOfHeaders ); }); + test(`(Step 4) CATALOG_FILE_INFO_RESPONSE.headers describe every column once | `, () => { + const columnIndices = CatalogFileInfoAck.headers.map((header) => header.columnIndex); + expect(columnIndices.slice().sort((a, b) => a - b)).toEqual( + Array.from({ length: assertItem.catalogFileInfoResponse[index].lengthOfHeaders }, (_, i) => i) + ); + CatalogFileInfoAck.headers.forEach((header) => { + expect(header.name).not.toEqual(''); + expect(header.dataType).not.toEqual(CARTA.ColumnType.UnsupportedType); + }); + }); + + let CatalogFileAck: CARTA.IOpenCatalogFileAck; test( `(Step 5) Request CatalogFile & check CatalogFileAck | `, async () => { - let CatalogFileAck = await msgController.loadCatalogFile( + CatalogFileAck = await msgController.loadCatalogFile( assertItem.openCatalogFile[index].directory, assertItem.openCatalogFile[index].name, assertItem.openCatalogFile[index].fileId, @@ -297,9 +350,8 @@ assertItem.catalogFileInfoReq.map((data, index) => { expect(CatalogFileAck.dataSize).toEqual(assertItem.openCatalogFileAck[index].dataSize); expect(CatalogFileAck.fileId).toEqual(assertItem.openCatalogFileAck[index].fileId); expect(CatalogFileAck.fileInfo.name).toEqual(assertItem.openCatalogFileAck[index].fileInfo.name); - if (CatalogFileAck.fileInfo.type) { - expect(CatalogFileAck.fileInfo.type).toEqual(assertItem.openCatalogFileAck[index].fileInfo.type); - } + // CatalogFileType.FITSTable is 0, so the type has to be compared unconditionally + expect(CatalogFileAck.fileInfo.type).toEqual(assertItem.openCatalogFileAck[index].fileInfo.type); expect(CatalogFileAck.fileInfo.fileSize.low).toEqual( assertItem.openCatalogFileAck[index].fileInfo.fileSize ); @@ -308,39 +360,85 @@ assertItem.catalogFileInfoReq.map((data, index) => { openCatalogLargeTimeout ); + test(`(Step 5) OPEN_CATALOG_FILE_ACK.preview_data holds ${assertItem.openCatalogFile[0].previewDataSize} rows of every column | `, () => { + const previewData = CatalogFileAck.previewData; + expect(Object.keys(previewData).length).toEqual(assertItem.openCatalogFileAck[index].lengthOfPreviewData); + Object.keys(previewData).forEach((key) => { + expect(columnRowCount(previewData[key])).toEqual(assertItem.openCatalogFile[index].previewDataSize); + }); + }); + + let CatalogFilterResponse: CARTA.ICatalogFilterResponse[]; test( - `(Step 6) Request CatalogFilter: Sorting & check CatalogFilterResponse | `, + `(Step 6) Request CatalogFilter & receive ${assertItem.catalogFilterResponse[0].numberOfResponses} streamed CatalogFilterResponse | `, async () => { - await msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[index]); - let CatalogFilterResponse = await Stream(CARTA.CatalogFilterResponse); - for (let i = 0; i < CatalogFilterResponse.length; i++) { - console.log( - `"${assertItem.catalogFileInfoReq[index].name}" CatalogFilterResponse progress :`, - CatalogFilterResponse[i].progress - ); - } - let lastCatalogFilterResponse = CatalogFilterResponse.slice(-1)[0]; - expect(Object.keys(lastCatalogFilterResponse.columns).length).toEqual( - assertItem.catalogFilterResponse[index].lengthOfColumns + // The stream has to be subscribed before the request is sent, otherwise the + // first chunks are dropped and only the tail of the stream is asserted. + const catalogFilterStream = Stream(CARTA.CatalogFilterResponse); + msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[index]); + CatalogFilterResponse = await catalogFilterStream; + console.log( + `"${assertItem.catalogFileInfoReq[index].name}" CatalogFilterResponse progress :`, + CatalogFilterResponse.map((response) => response.progress) ); - expect(lastCatalogFilterResponse.fileid).toEqual(assertItem.catalogFilterResponse[index].fileId); - expect(lastCatalogFilterResponse.subsetDataSize).toEqual( - assertItem.catalogFilterResponse[index].subsetDataSize - ); - expect(lastCatalogFilterResponse.subsetEndIndex).toEqual( - assertItem.catalogFilterResponse[index].subsetEndIndex - ); - expect(lastCatalogFilterResponse.filterDataSize).toEqual( - assertItem.catalogFilterResponse[index].filterDataSize - ); - expect(lastCatalogFilterResponse.requestEndIndex).toEqual( - assertItem.catalogFilterResponse[index].requestEndIndex - ); - expect(lastCatalogFilterResponse.progress).toEqual(assertItem.catalogFilterResponse[index].progress); + expect(CatalogFilterResponse.length).toEqual(assertItem.catalogFilterResponse[index].numberOfResponses); }, openCatalogLargeTimeout ); + test(`(Step 6) every CatalogFilterResponse reports the requested catalog and the whole filtered table | `, () => { + CatalogFilterResponse.forEach((response) => { + expect(response.fileId).toEqual(assertItem.catalogFilterResponse[index].fileId); + expect(response.filterDataSize).toEqual(assertItem.catalogFilterResponse[index].filterDataSize); + expect(response.requestEndIndex).toEqual(assertItem.catalogFilterResponse[index].requestEndIndex); + expect(Object.keys(response.columns)).toEqual( + assertItem.catalogFilterReq[index].columnIndices.map((columnIndex) => `${columnIndex}`) + ); + }); + }); + + test(`(Step 6) the streamed chunks cover the requested subset without a gap or an overlap | `, () => { + const maxChunkSize = assertItem.catalogFilterResponse[index].maxChunkSize; + let expectedStartIndex = assertItem.catalogFilterReq[index].subsetStartIndex; + let remainingRows = assertItem.catalogFilterReq[index].subsetDataSize; + CatalogFilterResponse.forEach((response) => { + const expectedChunkSize = Math.min(maxChunkSize, remainingRows); + expect(response.subsetDataSize).toEqual(expectedChunkSize); + // The frontend derives the row offset of a chunk from these two fields, so a + // chunk has to continue exactly where the previous one ended. + expect(response.subsetEndIndex - response.subsetDataSize).toEqual(expectedStartIndex); + Object.keys(response.columns).forEach((key) => { + expect(columnRowCount(response.columns[key])).toEqual(expectedChunkSize); + }); + expectedStartIndex += expectedChunkSize; + remainingRows -= expectedChunkSize; + }); + expect(remainingRows).toEqual(0); + }); + + test(`(Step 6) the progress increases and reaches 1 only in the last CatalogFilterResponse | `, () => { + const progresses = CatalogFilterResponse.map((response) => response.progress); + progresses.slice(0, -1).forEach((progress, i) => { + expect(progress).toBeGreaterThan(0); + expect(progress).toBeLessThan(1); + expect(progresses[i + 1]).toBeGreaterThan(progress); + }); + expect(progresses.slice(-1)[0]).toEqual(assertItem.catalogFilterResponse[index].progress); + }); + + test(`(Step 6) the last CatalogFilterResponse ends the requested subset | `, () => { + const lastCatalogFilterResponse = CatalogFilterResponse.slice(-1)[0]; + expect(Object.keys(lastCatalogFilterResponse.columns).length).toEqual( + assertItem.catalogFilterResponse[index].lengthOfColumns + ); + expect(lastCatalogFilterResponse.subsetDataSize).toEqual( + assertItem.catalogFilterResponse[index].subsetDataSize + ); + expect(lastCatalogFilterResponse.subsetEndIndex).toEqual( + assertItem.catalogFilterResponse[index].subsetEndIndex + ); + }); + afterAll(() => msgController.closeConnection()); }); }); From 23143c7b029abc6c166770dd122c1c5b1f7c3960 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 14 Aug 2026 13:06:47 +0800 Subject: [PATCH 2/4] Verify the filtered and sorted rows in the general catalog test --- ICD_test_stages/catalog.tests | 1 + docs/source/catalog.rst | 27 ++- src/test/CATALOG_GENERAL.test.ts | 275 ++++++++++++++++++++++--------- 3 files changed, 224 insertions(+), 79 deletions(-) diff --git a/ICD_test_stages/catalog.tests b/ICD_test_stages/catalog.tests index ef4012ed..42a5b550 100644 --- a/ICD_test_stages/catalog.tests +++ b/ICD_test_stages/catalog.tests @@ -1,2 +1,3 @@ src/test/CATALOG_GENERAL.test.ts src/test/CATALOG_FITS_VOT.test.ts +src/test/CATALOG_LARGE.test.ts diff --git a/docs/source/catalog.rst b/docs/source/catalog.rst index 701f2d06..28d485bb 100644 --- a/docs/source/catalog.rst +++ b/docs/source/catalog.rst @@ -128,6 +128,10 @@ This test verifies the general catalog workflow: listing, file info, opening, an - CATALOG_FILE_INFO_RESPONSE.file_info.file_size = 113559 - Length of headers = 235 + - CATALOG_FILE_INFO_RESPONSE.file_info.description reports the file name, "Column Count: 235", and the coordinate system of this catalog: "Coordinate System: FK5", "Epoch: J2000" and "Equinox: 2000" + + - The headers place the sorted and filtered columns ("RA_d" and "OTYPE_S") inside the requested column_indices, since the requests name a column while the response keys its columns by index + 8. Frontend sends: **OPEN_CATALOG_FILE** (``OpenCatalogFile``) .. code-block:: protobuf @@ -145,8 +149,13 @@ This test verifies the general catalog workflow: listing, file info, opening, an - OPEN_CATALOG_FILE_ACK.data_size = 29 - OPEN_CATALOG_FILE_ACK.file_id = 1 - OPEN_CATALOG_FILE_ACK.file_info.name = "artificial_catalog_J2000.xml" + - OPEN_CATALOG_FILE_ACK.file_info.type = VOTable - Length of headers = 235 + - OPEN_CATALOG_FILE_ACK.preview_data has one entry per column (235), and every entry holds 29 rows: the request asks for 50 preview rows, which is more than the table holds, so the backend returns the whole table instead + +Every request below returns the whole subset in a single CATALOG_FILTER_RESPONSE, since the table holds only 29 rows. Each response reports file_id = 1, columns keyed by the requested column_indices, and one value per row of subset_data_size in every column. + **Case 1: Sort by column** 10. Frontend sends: **CATALOG_FILTER_REQUEST** (``CatalogFilterRequest``) @@ -163,22 +172,26 @@ This test verifies the general catalog workflow: listing, file info, opening, an :red-text:`Check 5:` the CATALOG_FILTER_RESPONSE should satisfy: - Length of columns = 10, progress = 1 - - subsetDataSize = 29, filterDataSize = 29 + - subsetDataSize = 29, filterDataSize = 29 (no filter is applied, so the whole table is returned) + + - The returned "RA_d" values are in ascending order -**Case 2: Filter by number (RA_d > 160)** +**Case 2: Filter by number (RA_d >= 160)** 11. Frontend sends: **CATALOG_FILTER_REQUEST** with numeric filter .. code-block:: protobuf file_id = 1 - filter_configs = [{column_name: "RA_d", comparison_operator: 5, value: 160}] + filter_configs = [{column_name: "RA_d", comparison_operator: GreaterOrEqual, value: 160}] subset_data_size = 29 :red-text:`Check 6:` the CATALOG_FILTER_RESPONSE should satisfy: - subsetDataSize = 26, filterDataSize = 26 (3 rows filtered out) + - Every returned "RA_d" value is greater than or equal to 160, and the number of returned rows equals the number of rows of the unfiltered table which meet that condition + **Case 3: Filter by string (OTYPE_S contains "Star")** 12. Frontend sends: **CATALOG_FILTER_REQUEST** with string filter @@ -193,6 +206,8 @@ This test verifies the general catalog workflow: listing, file info, opening, an - subsetDataSize = 24, filterDataSize = 24 (5 rows filtered out) + - Every returned "OTYPE_S" value contains "Star", and the number of returned rows equals the number of rows of the unfiltered table which meet that condition. The backend matches a case-sensitive substring rather than the whole value + **Case 4: Combined filter + sort** 13. Frontend sends: **CATALOG_FILTER_REQUEST** with both string and numeric filters, plus sorting @@ -202,16 +217,18 @@ This test verifies the general catalog workflow: listing, file info, opening, an file_id = 1 filter_configs = [ {column_name: "OTYPE_S", sub_string: "Star"}, - {column_name: "RA_d", comparison_operator: 5, value: 160} + {column_name: "RA_d", comparison_operator: GreaterOrEqual, value: 160} ] sort_column = "RA_d" - sorting_type = 0 + sorting_type = Ascending subset_data_size = 29 :red-text:`Check 8:` the CATALOG_FILTER_RESPONSE should satisfy: - subsetDataSize = 23, filterDataSize = 23 (6 rows filtered out by combined filters) + - Every returned row satisfies both filters at once, the returned "RA_d" values are in ascending order, and the two filters together keep fewer rows than either one alone + CATALOG_FITS_VOT ~~~~~~~~~~~~~~~~ diff --git a/src/test/CATALOG_GENERAL.test.ts b/src/test/CATALOG_GENERAL.test.ts index 58d04a1c..41694c89 100644 --- a/src/test/CATALOG_GENERAL.test.ts +++ b/src/test/CATALOG_GENERAL.test.ts @@ -11,15 +11,50 @@ let readFileTimeout: number = config.timeout.readFile; interface ICatalogFileInfoResponseExt extends CARTA.ICatalogFileInfoResponse { lengthOfHeaders: number; + descriptionKeywords: string[]; } interface IOpenCatalogFileAckExt extends CARTA.IOpenCatalogFileAck { lengthOfHeaders: number; + lengthOfPreviewData: number; } interface ICatalogFilterResponseExt extends CARTA.ICatalogFilterResponse { lengthOfColumns: number; - fileid: number; + numberOfResponses: number; +} + +// ColumnData carries every type other than String as binary, so the number of rows in a +// column has to be derived from the byte length of its payload. +const bytesPerElement = new Map([ + [CARTA.ColumnType.Uint8, 1], + [CARTA.ColumnType.Int8, 1], + [CARTA.ColumnType.Bool, 1], + [CARTA.ColumnType.Uint16, 2], + [CARTA.ColumnType.Int16, 2], + [CARTA.ColumnType.Uint32, 4], + [CARTA.ColumnType.Int32, 4], + [CARTA.ColumnType.Float, 4], + [CARTA.ColumnType.Uint64, 8], + [CARTA.ColumnType.Int64, 8], + [CARTA.ColumnType.Double, 8], +]); + +function columnRowCount(column: CARTA.IColumnData): number { + if (column.dataType === CARTA.ColumnType.String) { + return column.stringData!.length; + } + return column.binaryData!.length / bytesPerElement.get(column.dataType!)!; +} + +function stringColumn(column: CARTA.IColumnData): string[] { + expect(column.dataType).toEqual(CARTA.ColumnType.String); + return column.stringData!; +} + +function doubleColumn(column: CARTA.IColumnData): number[] { + expect(column.dataType).toEqual(CARTA.ColumnType.Double); + return Array.from(new Float64Array(column.binaryData!.slice().buffer)); } interface AssertItem { @@ -88,14 +123,16 @@ let assertItem: AssertItem = { imageBounds: {}, regionId: null, sortColumn: 'RA_d', - sortingType: 0, + sortingType: CARTA.SortingType.Ascending, subsetDataSize: 29, subsetStartIndex: 0, }, { columnIndices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], fileId: 1, - filterConfigs: [{ columnName: 'RA_d', comparisonOperator: 5, value: 160 }], + filterConfigs: [ + { columnName: 'RA_d', comparisonOperator: CARTA.ComparisonOperator.GreaterOrEqual, value: 160 }, + ], imageBounds: { xColumnName: null, yColumnName: null, @@ -125,7 +162,7 @@ let assertItem: AssertItem = { fileId: 1, filterConfigs: [ { columnName: 'OTYPE_S', subString: 'Star' }, - { columnName: 'RA_d', comparisonOperator: 5, value: 160 }, + { columnName: 'RA_d', comparisonOperator: CARTA.ComparisonOperator.GreaterOrEqual, value: 160 }, ], imageBounds: { xColumnName: null, @@ -133,7 +170,7 @@ let assertItem: AssertItem = { }, regionId: null, sortColumn: 'RA_d', - sortingType: 0, + sortingType: CARTA.SortingType.Ascending, subsetDataSize: 29, subsetStartIndex: 0, }, @@ -146,27 +183,37 @@ let assertItem: AssertItem = { catalogFileInfoResponse: { fileInfo: { name: 'artificial_catalog_J2000.xml', - type: 1, + type: CARTA.CatalogFileType.VOTable, fileSize: 113559, }, success: true, lengthOfHeaders: 235, + descriptionKeywords: [ + 'Name: artificial_catalog_J2000.xml', + 'Column Count: 235', + 'Coordinate System: FK5', + 'Epoch: J2000', + 'Equinox: 2000', + ], }, openCatalogFileAck: { dataSize: 29, fileId: 1, fileInfo: { name: 'artificial_catalog_J2000.xml', - type: 1, + type: CARTA.CatalogFileType.VOTable, fileSize: 113559, }, lengthOfHeaders: 235, + lengthOfPreviewData: 235, success: true, }, + // The table holds 29 rows, so every subset fits in one CatalogFilterResponse catalogFilterResponse: [ { lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 1, subsetDataSize: 29, subsetEndIndex: 29, filterDataSize: 29, @@ -175,7 +222,8 @@ let assertItem: AssertItem = { }, { lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 1, subsetDataSize: 26, subsetEndIndex: 26, filterDataSize: 26, @@ -184,7 +232,8 @@ let assertItem: AssertItem = { }, { lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 1, subsetDataSize: 24, subsetEndIndex: 24, filterDataSize: 24, @@ -193,7 +242,8 @@ let assertItem: AssertItem = { }, { lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 1, subsetDataSize: 23, subsetEndIndex: 23, filterDataSize: 23, @@ -203,6 +253,37 @@ let assertItem: AssertItem = { ], }; +function assertCatalogFilterResponse( + responses: CARTA.ICatalogFilterResponse[], + expected: ICatalogFilterResponseExt, + requestedColumnIndices: number[] +) { + expect(responses.length).toEqual(expected.numberOfResponses); + const response = responses[0]; + expect(response.fileId).toEqual(expected.fileId); + expect(Object.keys(response.columns).length).toEqual(expected.lengthOfColumns); + expect(Object.keys(response.columns)).toEqual(requestedColumnIndices.map((columnIndex) => `${columnIndex}`)); + expect(response.subsetDataSize).toEqual(expected.subsetDataSize); + expect(response.subsetEndIndex).toEqual(expected.subsetEndIndex); + expect(response.filterDataSize).toEqual(expected.filterDataSize); + expect(response.requestEndIndex).toEqual(expected.requestEndIndex); + expect(response.progress).toEqual(expected.progress); + Object.keys(response.columns).forEach((key) => { + expect(columnRowCount(response.columns[key])).toEqual(expected.subsetDataSize); + }); +} + +function requestCatalogFilter( + msgController: MessageController, + filterRequest: CARTA.ICatalogFilterRequest +): Promise { + // The stream has to be subscribed before the request is sent, otherwise the response + // is dropped and the promise never settles. + const catalogFilterStream = Stream(CARTA.CatalogFilterResponse); + msgController.setCatalogFilterRequest(filterRequest); + return catalogFilterStream; +} + let basepath: string; describe('Test for general CATALOG related messages:', () => { const msgController = MessageController.Instance; @@ -271,8 +352,9 @@ describe('Test for general CATALOG related messages:', () => { ); }); + let CatalogFileInfoAck: CARTA.ICatalogFileInfoResponse; test(`(Step 4) Request CatalogFileInfo & check CatalogFileInfoAck | `, async () => { - let CatalogFileInfoAck = await msgController.getCatalogFileInfo( + CatalogFileInfoAck = await msgController.getCatalogFileInfo( assertItem.catalogFileInfoReq.directory, assertItem.catalogFileInfoReq.name ); @@ -282,11 +364,33 @@ describe('Test for general CATALOG related messages:', () => { expect(CatalogFileInfoAck.fileInfo.fileSize.low).toEqual( assertItem.catalogFileInfoResponse.fileInfo.fileSize ); + assertItem.catalogFileInfoResponse.descriptionKeywords.forEach((keyword) => { + expect(CatalogFileInfoAck.fileInfo.description).toContain(keyword); + }); expect(CatalogFileInfoAck.headers.length).toEqual(assertItem.catalogFileInfoResponse.lengthOfHeaders); }); + // The filter and the sort are requested by column name, while the response keys its + // columns by index, so the two are tied together through the headers. + let filterColumnIndices: Map; + test(`(Step 4) the headers place the filtered columns inside the requested column_indices | `, () => { + filterColumnIndices = new Map( + CatalogFileInfoAck.headers.map((header) => [header.name, header.columnIndex]) + ); + const filteredColumnNames = [ + assertItem.catalogFilterReq[0].sortColumn, + assertItem.catalogFilterReq[1].filterConfigs[0].columnName, + assertItem.catalogFilterReq[2].filterConfigs[0].columnName, + ]; + filteredColumnNames.forEach((columnName) => { + expect(filterColumnIndices.get(columnName)).toBeDefined(); + expect(assertItem.catalogFilterReq[0].columnIndices).toContain(filterColumnIndices.get(columnName)); + }); + }); + + let CatalogFileAck: CARTA.IOpenCatalogFileAck; test(`(Step 5) Request CatalogFile & check CatalogFileAck | `, async () => { - let CatalogFileAck = await msgController.loadCatalogFile( + CatalogFileAck = await msgController.loadCatalogFile( assertItem.openCatalogFile.directory, assertItem.openCatalogFile.name, assertItem.openCatalogFile.fileId, @@ -301,86 +405,109 @@ describe('Test for general CATALOG related messages:', () => { expect(CatalogFileAck.headers.length).toEqual(assertItem.openCatalogFileAck.lengthOfHeaders); }); + test(`(Step 5) OPEN_CATALOG_FILE_ACK.preview_data is clamped to the ${assertItem.openCatalogFileAck.dataSize} rows of the table | `, () => { + const previewData = CatalogFileAck.previewData; + expect(Object.keys(previewData).length).toEqual(assertItem.openCatalogFileAck.lengthOfPreviewData); + // preview_data_size asks for more rows than the table holds, so the backend + // returns the whole table instead. + expect(assertItem.openCatalogFile.previewDataSize).toBeGreaterThan(assertItem.openCatalogFileAck.dataSize); + Object.keys(previewData).forEach((key) => { + expect(columnRowCount(previewData[key])).toEqual(assertItem.openCatalogFileAck.dataSize); + }); + }); + + let SortedResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 6) Request CatalogFilter: Sorting & check CatalogFilterResponse | `, async () => { - await msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[0]); - let CatalogFilterResponse = await Stream(CARTA.CatalogFilterResponse); - expect(Object.keys(CatalogFilterResponse[0].columns).length).toEqual( - assertItem.catalogFilterResponse[0].lengthOfColumns + SortedResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[0]); + assertCatalogFilterResponse( + SortedResponse, + assertItem.catalogFilterResponse[0], + assertItem.catalogFilterReq[0].columnIndices ); - expect(CatalogFilterResponse[0].fileid).toEqual(assertItem.catalogFilterResponse[0].fileId); - expect(CatalogFilterResponse[0].subsetDataSize).toEqual(assertItem.catalogFilterResponse[0].subsetDataSize); - expect(CatalogFilterResponse[0].subsetEndIndex).toEqual(assertItem.catalogFilterResponse[0].subsetEndIndex); - expect(CatalogFilterResponse[0].filterDataSize).toEqual(assertItem.catalogFilterResponse[0].filterDataSize); - expect(CatalogFilterResponse[0].requestEndIndex).toEqual( - assertItem.catalogFilterResponse[0].requestEndIndex - ); - expect(CatalogFilterResponse[0].progress).toEqual(assertItem.catalogFilterResponse[0].progress); }); + test(`(Step 6) the rows are sorted by "${assertItem.catalogFilterReq[0].sortColumn}" in ascending order | `, () => { + const sortColumnIndex = filterColumnIndices.get(assertItem.catalogFilterReq[0].sortColumn); + const sortValues = doubleColumn(SortedResponse[0].columns[sortColumnIndex]); + expect(sortValues).toEqual([...sortValues].sort((a, b) => a - b)); + }); + + let NumberFilteredResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 7) Request CatalogFilter: Filter(number) & check CatalogFilterResponse | `, async () => { - await msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[1]); - let CatalogFilterResponse2 = await Stream(CARTA.CatalogFilterResponse); - expect(Object.keys(CatalogFilterResponse2[0].columns).length).toEqual( - assertItem.catalogFilterResponse[1].lengthOfColumns - ); - expect(CatalogFilterResponse2[0].fileid).toEqual(assertItem.catalogFilterResponse[1].fileId); - expect(CatalogFilterResponse2[0].subsetDataSize).toEqual( - assertItem.catalogFilterResponse[1].subsetDataSize - ); - expect(CatalogFilterResponse2[0].subsetEndIndex).toEqual( - assertItem.catalogFilterResponse[1].subsetEndIndex + NumberFilteredResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[1]); + assertCatalogFilterResponse( + NumberFilteredResponse, + assertItem.catalogFilterResponse[1], + assertItem.catalogFilterReq[1].columnIndices ); - expect(CatalogFilterResponse2[0].filterDataSize).toEqual( - assertItem.catalogFilterResponse[1].filterDataSize - ); - expect(CatalogFilterResponse2[0].requestEndIndex).toEqual( - assertItem.catalogFilterResponse[1].requestEndIndex + }); + + test(`(Step 7) every returned row satisfies the number filter | `, () => { + const filterConfig = assertItem.catalogFilterReq[1].filterConfigs[0]; + const filterValues = doubleColumn( + NumberFilteredResponse[0].columns[filterColumnIndices.get(filterConfig.columnName)] ); - expect(CatalogFilterResponse2[0].progress).toEqual(assertItem.catalogFilterResponse[1].progress); + filterValues.forEach((value) => { + expect(value).toBeGreaterThanOrEqual(filterConfig.value); + }); + // The unfiltered table is the sorted response of step 6, so the rows the filter + // removed have to be the ones below the threshold. + const allValues = doubleColumn(SortedResponse[0].columns[filterColumnIndices.get(filterConfig.columnName)]); + expect(filterValues.length).toEqual(allValues.filter((value) => value >= filterConfig.value).length); }); + let StringFilteredResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 8) Request CatalogFilter: Filter(string) & check CatalogFilterResponse | `, async () => { - await msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[2]); - let CatalogFilterResponse3 = await Stream(CARTA.CatalogFilterResponse); - expect(Object.keys(CatalogFilterResponse3[0].columns).length).toEqual( - assertItem.catalogFilterResponse[2].lengthOfColumns - ); - expect(CatalogFilterResponse3[0].fileid).toEqual(assertItem.catalogFilterResponse[2].fileId); - expect(CatalogFilterResponse3[0].subsetDataSize).toEqual( - assertItem.catalogFilterResponse[2].subsetDataSize - ); - expect(CatalogFilterResponse3[0].subsetEndIndex).toEqual( - assertItem.catalogFilterResponse[2].subsetEndIndex + StringFilteredResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[2]); + assertCatalogFilterResponse( + StringFilteredResponse, + assertItem.catalogFilterResponse[2], + assertItem.catalogFilterReq[2].columnIndices ); - expect(CatalogFilterResponse3[0].filterDataSize).toEqual( - assertItem.catalogFilterResponse[2].filterDataSize + }); + + test(`(Step 8) every returned row satisfies the string filter | `, () => { + const filterConfig = assertItem.catalogFilterReq[2].filterConfigs[0]; + const filterValues = stringColumn( + StringFilteredResponse[0].columns[filterColumnIndices.get(filterConfig.columnName)] ); - expect(CatalogFilterResponse3[0].requestEndIndex).toEqual( - assertItem.catalogFilterResponse[2].requestEndIndex + filterValues.forEach((value) => { + expect(value).toContain(filterConfig.subString); + }); + const allValues = stringColumn(SortedResponse[0].columns[filterColumnIndices.get(filterConfig.columnName)]); + expect(filterValues.length).toEqual( + allValues.filter((value) => value.includes(filterConfig.subString)).length ); - expect(CatalogFilterResponse3[0].progress).toEqual(assertItem.catalogFilterResponse[2].progress); }); + let SortedAndFilteredResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 9) Request CatalogFilter: Sorting when Filter(string+number) is applied & check CatalogFilterResponse | `, async () => { - await msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[3]); - let CatalogFilterResponse4 = await Stream(CARTA.CatalogFilterResponse); - expect(Object.keys(CatalogFilterResponse4[0].columns).length).toEqual( - assertItem.catalogFilterResponse[3].lengthOfColumns + SortedAndFilteredResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[3]); + assertCatalogFilterResponse( + SortedAndFilteredResponse, + assertItem.catalogFilterResponse[3], + assertItem.catalogFilterReq[3].columnIndices ); - expect(CatalogFilterResponse4[0].fileid).toEqual(assertItem.catalogFilterResponse[3].fileId); - expect(CatalogFilterResponse4[0].subsetDataSize).toEqual( - assertItem.catalogFilterResponse[3].subsetDataSize - ); - expect(CatalogFilterResponse4[0].subsetEndIndex).toEqual( - assertItem.catalogFilterResponse[3].subsetEndIndex - ); - expect(CatalogFilterResponse4[0].filterDataSize).toEqual( - assertItem.catalogFilterResponse[3].filterDataSize + }); + + test(`(Step 9) the returned rows satisfy both filters and are sorted by "${assertItem.catalogFilterReq[3].sortColumn}" | `, () => { + const [stringFilterConfig, numberFilterConfig] = assertItem.catalogFilterReq[3].filterConfigs; + const stringValues = stringColumn( + SortedAndFilteredResponse[0].columns[filterColumnIndices.get(stringFilterConfig.columnName)] ); - expect(CatalogFilterResponse4[0].requestEndIndex).toEqual( - assertItem.catalogFilterResponse[3].requestEndIndex + const numberValues = doubleColumn( + SortedAndFilteredResponse[0].columns[filterColumnIndices.get(numberFilterConfig.columnName)] ); - expect(CatalogFilterResponse4[0].progress).toEqual(assertItem.catalogFilterResponse[3].progress); + stringValues.forEach((value) => { + expect(value).toContain(stringFilterConfig.subString); + }); + numberValues.forEach((value) => { + expect(value).toBeGreaterThanOrEqual(numberFilterConfig.value); + }); + expect(numberValues).toEqual([...numberValues].sort((a, b) => a - b)); + // Both filters together keep fewer rows than either one alone + expect(SortedAndFilteredResponse[0].filterDataSize).toBeLessThan(NumberFilteredResponse[0].filterDataSize); + expect(SortedAndFilteredResponse[0].filterDataSize).toBeLessThan(StringFilteredResponse[0].filterDataSize); }); afterAll(() => msgController.closeConnection()); From 432d25cf378143614b1ce42a36d5eb35bbd926c2 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 14 Aug 2026 13:15:54 +0800 Subject: [PATCH 3/4] Verify the progressively loaded rows in the large catalog test --- docs/source/catalog.rst | 15 ++- src/test/CATALOG_LARGE.test.ts | 184 ++++++++++++++++++++++++--------- 2 files changed, 147 insertions(+), 52 deletions(-) diff --git a/docs/source/catalog.rst b/docs/source/catalog.rst index 28d485bb..a452a469 100644 --- a/docs/source/catalog.rst +++ b/docs/source/catalog.rst @@ -389,10 +389,15 @@ This test verifies catalog operations with a large VOTable catalog (COSMOSOPTCAT subset_data_size = 918777 subset_start_index = 50 -:red-text:`Check 3:` the CATALOG_FILTER_RESPONSE should satisfy: + The 918,777 requested rows exceed the 100000 rows the backend puts in one message, so they + are streamed as 10 CATALOG_FILTER_RESPONSE messages. + +:red-text:`Check 3:` the streamed CATALOG_FILTER_RESPONSE should satisfy: - Should arrive within 100000 ms - - subsetDataSize = 18777, subsetEndIndex = 918827, filterDataSize = 918827, progress = 1 + - Exactly 10 messages are streamed, and the progress increases from message to message and equals 1 only in the last one + - Last message: subsetDataSize = 18777, subsetEndIndex = 918827, filterDataSize = 918827, progress = 1 + - Every column of a message holds one value per row of its subsetDataSize **Part 2: Progressive load of rows** @@ -411,7 +416,11 @@ This test verifies catalog operations with a large VOTable catalog (COSMOSOPTCAT - Request 1: subsetDataSize = 50, subsetEndIndex = 100, filterDataSize = 918827 - Request 2: subsetDataSize = 50, subsetEndIndex = 150, filterDataSize = 918827 - Request 3: subsetDataSize = 50, subsetEndIndex = 200, filterDataSize = 918827 - - All with progress = 1 + - All with progress = 1, in a single message, since a window of 50 rows fits in one message + + - A window ends at subset_start_index + subset_data_size, and filterDataSize stays at the 918827 rows of the table, so paging through the table does not change the number of rows it reports + + - The three windows return three different sets of rows, and each window returns exactly the rows which the whole table load of part 1 returned at the same position in the table. This is what makes progressive loading equivalent to loading the table at one time IMPORT_MULTIPLE_CATALOG ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/test/CATALOG_LARGE.test.ts b/src/test/CATALOG_LARGE.test.ts index 1f247d5d..d3c92f45 100644 --- a/src/test/CATALOG_LARGE.test.ts +++ b/src/test/CATALOG_LARGE.test.ts @@ -20,7 +20,72 @@ interface IOpenCatalogFileAckExt extends CARTA.IOpenCatalogFileAck { interface ICatalogFilterResponseExt extends CARTA.ICatalogFilterResponse { lengthOfColumns: number; - fileid: number; + numberOfResponses: number; +} + +// ColumnData carries every type other than String as binary, so a row of a column is a +// fixed number of bytes rather than an element of an array. +const bytesPerElement = new Map([ + [CARTA.ColumnType.Uint8, 1], + [CARTA.ColumnType.Int8, 1], + [CARTA.ColumnType.Bool, 1], + [CARTA.ColumnType.Uint16, 2], + [CARTA.ColumnType.Int16, 2], + [CARTA.ColumnType.Uint32, 4], + [CARTA.ColumnType.Int32, 4], + [CARTA.ColumnType.Float, 4], + [CARTA.ColumnType.Uint64, 8], + [CARTA.ColumnType.Int64, 8], + [CARTA.ColumnType.Double, 8], +]); + +// Rows are compared as raw payload, which works for every column type without having to +// decode it. ProtobufProcessing cannot be used here because it needs the CARTACompute +// WASM global for the 64 bit types. +function columnSlice(column: CARTA.IColumnData, startRow: number, rowCount: number): (string | number)[] { + if (column.dataType === CARTA.ColumnType.String) { + return column.stringData!.slice(startRow, startRow + rowCount); + } + const elementSize = bytesPerElement.get(column.dataType!)!; + return Array.from(column.binaryData!.slice(startRow * elementSize, (startRow + rowCount) * elementSize)); +} + +function columnRowCount(column: CARTA.IColumnData): number { + if (column.dataType === CARTA.ColumnType.String) { + return column.stringData!.length; + } + return column.binaryData!.length / bytesPerElement.get(column.dataType!)!; +} + +function assertCatalogFilterResponse( + responses: CARTA.ICatalogFilterResponse[], + expected: ICatalogFilterResponseExt, + request: CARTA.ICatalogFilterRequest +) { + expect(responses.length).toEqual(expected.numberOfResponses); + const lastResponse = responses.slice(-1)[0]; + expect(lastResponse.fileId).toEqual(expected.fileId); + expect(Object.keys(lastResponse.columns).length).toEqual(expected.lengthOfColumns); + expect(Object.keys(lastResponse.columns)).toEqual(request.columnIndices.map((columnIndex) => `${columnIndex}`)); + expect(lastResponse.subsetDataSize).toEqual(expected.subsetDataSize); + expect(lastResponse.subsetEndIndex).toEqual(expected.subsetEndIndex); + expect(lastResponse.filterDataSize).toEqual(expected.filterDataSize); + expect(lastResponse.requestEndIndex).toEqual(expected.requestEndIndex); + expect(lastResponse.progress).toEqual(expected.progress); + Object.keys(lastResponse.columns).forEach((key) => { + expect(columnRowCount(lastResponse.columns[key])).toEqual(expected.subsetDataSize); + }); +} + +function requestCatalogFilter( + msgController: MessageController, + filterRequest: CARTA.ICatalogFilterRequest +): Promise { + // The stream has to be subscribed before the request is sent, otherwise the first + // responses are dropped. + const catalogFilterStream = Stream(CARTA.CatalogFilterResponse); + msgController.setCatalogFilterRequest(filterRequest); + return catalogFilterStream; } interface AssertItem { @@ -133,21 +198,23 @@ let assertItem: AssertItem = { subdirectories: [], }, catalogFileInfoResponse: { - fileInfo: { name: 'COSMOSOPTCAT.vot', type: 1, fileSize: 1631311089 }, + fileInfo: { name: 'COSMOSOPTCAT.vot', type: CARTA.CatalogFileType.VOTable, fileSize: 1631311089 }, success: true, lengthOfHeaders: 62, }, openCatalogFileAck: { dataSize: 918827, fileId: 1, - fileInfo: { name: 'COSMOSOPTCAT.vot', type: 1, fileSize: 1631311089 }, + fileInfo: { name: 'COSMOSOPTCAT.vot', type: CARTA.CatalogFileType.VOTable, fileSize: 1631311089 }, lengthOfHeaders: 62, success: true, }, catalogFilterResponse: [ { + // The whole table is streamed in chunks of at most 100000 rows lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 10, subsetDataSize: 18777, subsetEndIndex: 918827, filterDataSize: 918827, @@ -155,8 +222,10 @@ let assertItem: AssertItem = { progress: 1, }, { + // A window of 50 rows fits in one response lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 1, subsetDataSize: 50, subsetEndIndex: 100, filterDataSize: 918827, @@ -165,7 +234,8 @@ let assertItem: AssertItem = { }, { lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 1, subsetDataSize: 50, subsetEndIndex: 150, filterDataSize: 918827, @@ -174,7 +244,8 @@ let assertItem: AssertItem = { }, { lengthOfColumns: 10, - fileid: 1, + fileId: 1, + numberOfResponses: 1, subsetDataSize: 50, subsetEndIndex: 200, filterDataSize: 918827, @@ -184,6 +255,10 @@ let assertItem: AssertItem = { ], }; +// The first chunk of the whole-table load, kept so that the progressive windows of part 2 +// can be compared against the rows the bulk load returned for the same table positions. +let wholeTableFirstChunk: CARTA.ICatalogFilterResponse; + let basepath: string; describe('Test for large-size CATALOG: load whole table at one time', () => { const msgController = MessageController.Instance; @@ -283,39 +358,34 @@ describe('Test for large-size CATALOG: load whole table at one time', () => { openCatalogLargeTimeout ); + let WholeTableResponse: CARTA.ICatalogFilterResponse[]; test( `(Step 6) Request CatalogFilter: progress & check CatalogFilterResponse | `, async () => { - await msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[0]); - let CatalogFilterResponse = await Stream(CARTA.CatalogFilterResponse); - for (let i = 0; i < CatalogFilterResponse.length; i++) { - console.log( - `"${assertItem.catalogFileInfoReq.name}" CatalogFilterResponse progress :`, - CatalogFilterResponse[i].progress - ); - } - let lastCatalogFilterResponse = CatalogFilterResponse.slice(-1)[0]; - expect(Object.keys(lastCatalogFilterResponse.columns).length).toEqual( - assertItem.catalogFilterResponse[0].lengthOfColumns - ); - expect(lastCatalogFilterResponse.fileid).toEqual(assertItem.catalogFilterResponse[0].fileId); - expect(lastCatalogFilterResponse.subsetDataSize).toEqual( - assertItem.catalogFilterResponse[0].subsetDataSize - ); - expect(lastCatalogFilterResponse.subsetEndIndex).toEqual( - assertItem.catalogFilterResponse[0].subsetEndIndex - ); - expect(lastCatalogFilterResponse.filterDataSize).toEqual( - assertItem.catalogFilterResponse[0].filterDataSize + WholeTableResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[0]); + wholeTableFirstChunk = WholeTableResponse[0]; + console.log( + `"${assertItem.catalogFileInfoReq.name}" CatalogFilterResponse progress :`, + WholeTableResponse.map((response) => response.progress) ); - expect(lastCatalogFilterResponse.requestEndIndex).toEqual( - assertItem.catalogFilterResponse[0].requestEndIndex + assertCatalogFilterResponse( + WholeTableResponse, + assertItem.catalogFilterResponse[0], + assertItem.catalogFilterReq[0] ); - expect(lastCatalogFilterResponse.progress).toEqual(assertItem.catalogFilterResponse[0].progress); }, openCatalogLargeTimeout ); + test(`(Step 6) the progress increases and reaches 1 only in the last CatalogFilterResponse | `, () => { + const progresses = WholeTableResponse.map((response) => response.progress); + progresses.slice(0, -1).forEach((progress, i) => { + expect(progress).toBeGreaterThan(0); + expect(progress).toBeLessThan(1); + expect(progresses[i + 1]).toBeGreaterThan(progress); + }); + }); + afterAll(() => msgController.closeConnection()); }); @@ -408,34 +478,50 @@ describe('Test for large-size CATALOG: Progressive load of rows', () => { openCatalogLargeTimeout ); + let ProgressiveWindows: CARTA.ICatalogFilterResponse[] = []; for (let i = 1; i < 4; i++) { test( `(Step 6-${i - 1}) Request CatalogFilter: subsetStartIndex of ${assertItem.catalogFilterReq[i].subsetStartIndex} & check CatalogFilterResponse | `, async () => { - await msgController.setCatalogFilterRequest(assertItem.catalogFilterReq[i]); - let CatalogFilterResponse = await Stream(CARTA.CatalogFilterResponse); - let lastCatalogFilterResponse = CatalogFilterResponse.slice(-1)[0]; - expect(Object.keys(lastCatalogFilterResponse.columns).length).toEqual( - assertItem.catalogFilterResponse[i].lengthOfColumns - ); - expect(lastCatalogFilterResponse.fileid).toEqual(assertItem.catalogFilterResponse[i].fileId); - expect(lastCatalogFilterResponse.subsetDataSize).toEqual( - assertItem.catalogFilterResponse[i].subsetDataSize - ); - expect(lastCatalogFilterResponse.subsetEndIndex).toEqual( - assertItem.catalogFilterResponse[i].subsetEndIndex + const CatalogFilterResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[i]); + ProgressiveWindows[i] = CatalogFilterResponse[0]; + assertCatalogFilterResponse( + CatalogFilterResponse, + assertItem.catalogFilterResponse[i], + assertItem.catalogFilterReq[i] ); - expect(lastCatalogFilterResponse.filterDataSize).toEqual( - assertItem.catalogFilterResponse[i].filterDataSize + // A window ends where the request asked it to, whatever its start index is + expect(CatalogFilterResponse[0].subsetEndIndex).toEqual( + assertItem.catalogFilterReq[i].subsetStartIndex + assertItem.catalogFilterReq[i].subsetDataSize ); - expect(lastCatalogFilterResponse.requestEndIndex).toEqual( - assertItem.catalogFilterResponse[i].requestEndIndex - ); - expect(lastCatalogFilterResponse.progress).toEqual(assertItem.catalogFilterResponse[i].progress); + // Paging through the table does not change the number of rows it holds + expect(CatalogFilterResponse[0].filterDataSize).toEqual(assertItem.openCatalogFileAck.dataSize); }, openCatalogLargeTimeout ); } + test(`(Step 6) each window returns the rows the whole table load returned at the same position | `, () => { + expect(wholeTableFirstChunk).toBeDefined(); + const wholeTableStartIndex = assertItem.catalogFilterReq[0].subsetStartIndex; + for (let i = 1; i < 4; i++) { + const request = assertItem.catalogFilterReq[i]; + const offset = request.subsetStartIndex - wholeTableStartIndex; + Object.keys(ProgressiveWindows[i].columns).forEach((key) => { + expect(columnSlice(ProgressiveWindows[i].columns[key], 0, request.subsetDataSize)).toEqual( + columnSlice(wholeTableFirstChunk.columns[key], offset, request.subsetDataSize) + ); + }); + } + }); + + test(`(Step 6) the three windows return three different sets of rows | `, () => { + const firstColumnKey = Object.keys(ProgressiveWindows[1].columns)[0]; + const windowRows = [1, 2, 3].map((i) => + JSON.stringify(columnSlice(ProgressiveWindows[i].columns[firstColumnKey], 0, 50)) + ); + expect(new Set(windowRows).size).toEqual(windowRows.length); + }); + afterAll(() => msgController.closeConnection()); }); From d7eb66a7d97469361193ee243f05456bccf74d95 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 14 Aug 2026 13:21:43 +0800 Subject: [PATCH 4/4] Share the catalog filter helpers through MyClient --- src/test/CATALOG_FITS_VOT.test.ts | 33 +---------- src/test/CATALOG_GENERAL.test.ts | 96 ++++++------------------------- src/test/CATALOG_LARGE.test.ts | 82 +++----------------------- src/test/MyClient.ts | 94 +++++++++++++++++++++++++++++- 4 files changed, 123 insertions(+), 182 deletions(-) diff --git a/src/test/CATALOG_FITS_VOT.test.ts b/src/test/CATALOG_FITS_VOT.test.ts index 4ea3de69..531ca6c7 100644 --- a/src/test/CATALOG_FITS_VOT.test.ts +++ b/src/test/CATALOG_FITS_VOT.test.ts @@ -1,5 +1,5 @@ import { CARTA } from 'carta-protobuf'; -import { Stream } from './MyClient'; +import { Stream, columnRowCount, ICatalogFilterResponseExt } from './MyClient'; import { MessageController, ConnectionStatus } from './MessageController'; import config from './config.json'; @@ -20,36 +20,9 @@ interface IOpenCatalogFileAckExt extends CARTA.IOpenCatalogFileAck { lengthOfPreviewData: number; } -interface ICatalogFilterResponseExt extends CARTA.ICatalogFilterResponse { - lengthOfColumns: number; +interface ICatalogFilterResponseExtLocal extends ICatalogFilterResponseExt { // The backend streams the requested subset in chunks of TableController's max_chunk_size maxChunkSize: number; - numberOfResponses: number; -} - -// ColumnData carries every type other than String as binary, so the number of rows in a -// column has to be derived from the byte length of its payload. -const bytesPerElement = new Map([ - [CARTA.ColumnType.Uint8, 1], - [CARTA.ColumnType.Int8, 1], - [CARTA.ColumnType.Bool, 1], - [CARTA.ColumnType.Uint16, 2], - [CARTA.ColumnType.Int16, 2], - [CARTA.ColumnType.Uint32, 4], - [CARTA.ColumnType.Int32, 4], - [CARTA.ColumnType.Float, 4], - [CARTA.ColumnType.Uint64, 8], - [CARTA.ColumnType.Int64, 8], - [CARTA.ColumnType.Double, 8], -]); - -function columnRowCount(column: CARTA.IColumnData): number { - if (column.dataType === CARTA.ColumnType.String) { - return column.stringData!.length; - } - const elementSize = bytesPerElement.get(column.dataType!)!; - expect(elementSize).toBeDefined(); - return column.binaryData!.length / elementSize; } interface AssertItem { @@ -66,7 +39,7 @@ interface AssertItem { openCatalogFile: CARTA.IOpenCatalogFile[]; openCatalogFileAck: IOpenCatalogFileAckExt[]; catalogFilterReq: CARTA.ICatalogFilterRequest[]; - catalogFilterResponse: ICatalogFilterResponseExt[]; + catalogFilterResponse: ICatalogFilterResponseExtLocal[]; } let assertItem: AssertItem = { diff --git a/src/test/CATALOG_GENERAL.test.ts b/src/test/CATALOG_GENERAL.test.ts index 41694c89..b8659089 100644 --- a/src/test/CATALOG_GENERAL.test.ts +++ b/src/test/CATALOG_GENERAL.test.ts @@ -1,5 +1,14 @@ import { CARTA } from 'carta-protobuf'; -import { checkConnection, Stream } from './MyClient'; +import { + checkConnection, + Stream, + columnRowCount, + stringColumn, + doubleColumn, + assertCatalogFilterResponse, + requestCatalogFilter, + ICatalogFilterResponseExt, +} from './MyClient'; import { MessageController } from './MessageController'; import config from './config.json'; @@ -19,44 +28,6 @@ interface IOpenCatalogFileAckExt extends CARTA.IOpenCatalogFileAck { lengthOfPreviewData: number; } -interface ICatalogFilterResponseExt extends CARTA.ICatalogFilterResponse { - lengthOfColumns: number; - numberOfResponses: number; -} - -// ColumnData carries every type other than String as binary, so the number of rows in a -// column has to be derived from the byte length of its payload. -const bytesPerElement = new Map([ - [CARTA.ColumnType.Uint8, 1], - [CARTA.ColumnType.Int8, 1], - [CARTA.ColumnType.Bool, 1], - [CARTA.ColumnType.Uint16, 2], - [CARTA.ColumnType.Int16, 2], - [CARTA.ColumnType.Uint32, 4], - [CARTA.ColumnType.Int32, 4], - [CARTA.ColumnType.Float, 4], - [CARTA.ColumnType.Uint64, 8], - [CARTA.ColumnType.Int64, 8], - [CARTA.ColumnType.Double, 8], -]); - -function columnRowCount(column: CARTA.IColumnData): number { - if (column.dataType === CARTA.ColumnType.String) { - return column.stringData!.length; - } - return column.binaryData!.length / bytesPerElement.get(column.dataType!)!; -} - -function stringColumn(column: CARTA.IColumnData): string[] { - expect(column.dataType).toEqual(CARTA.ColumnType.String); - return column.stringData!; -} - -function doubleColumn(column: CARTA.IColumnData): number[] { - expect(column.dataType).toEqual(CARTA.ColumnType.Double); - return Array.from(new Float64Array(column.binaryData!.slice().buffer)); -} - interface AssertItem { registerViewer: CARTA.IRegisterViewer; filelist: CARTA.IFileListRequest; @@ -253,37 +224,6 @@ let assertItem: AssertItem = { ], }; -function assertCatalogFilterResponse( - responses: CARTA.ICatalogFilterResponse[], - expected: ICatalogFilterResponseExt, - requestedColumnIndices: number[] -) { - expect(responses.length).toEqual(expected.numberOfResponses); - const response = responses[0]; - expect(response.fileId).toEqual(expected.fileId); - expect(Object.keys(response.columns).length).toEqual(expected.lengthOfColumns); - expect(Object.keys(response.columns)).toEqual(requestedColumnIndices.map((columnIndex) => `${columnIndex}`)); - expect(response.subsetDataSize).toEqual(expected.subsetDataSize); - expect(response.subsetEndIndex).toEqual(expected.subsetEndIndex); - expect(response.filterDataSize).toEqual(expected.filterDataSize); - expect(response.requestEndIndex).toEqual(expected.requestEndIndex); - expect(response.progress).toEqual(expected.progress); - Object.keys(response.columns).forEach((key) => { - expect(columnRowCount(response.columns[key])).toEqual(expected.subsetDataSize); - }); -} - -function requestCatalogFilter( - msgController: MessageController, - filterRequest: CARTA.ICatalogFilterRequest -): Promise { - // The stream has to be subscribed before the request is sent, otherwise the response - // is dropped and the promise never settles. - const catalogFilterStream = Stream(CARTA.CatalogFilterResponse); - msgController.setCatalogFilterRequest(filterRequest); - return catalogFilterStream; -} - let basepath: string; describe('Test for general CATALOG related messages:', () => { const msgController = MessageController.Instance; @@ -418,11 +358,11 @@ describe('Test for general CATALOG related messages:', () => { let SortedResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 6) Request CatalogFilter: Sorting & check CatalogFilterResponse | `, async () => { - SortedResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[0]); + SortedResponse = await requestCatalogFilter(assertItem.catalogFilterReq[0]); assertCatalogFilterResponse( SortedResponse, assertItem.catalogFilterResponse[0], - assertItem.catalogFilterReq[0].columnIndices + assertItem.catalogFilterReq[0] ); }); @@ -434,11 +374,11 @@ describe('Test for general CATALOG related messages:', () => { let NumberFilteredResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 7) Request CatalogFilter: Filter(number) & check CatalogFilterResponse | `, async () => { - NumberFilteredResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[1]); + NumberFilteredResponse = await requestCatalogFilter(assertItem.catalogFilterReq[1]); assertCatalogFilterResponse( NumberFilteredResponse, assertItem.catalogFilterResponse[1], - assertItem.catalogFilterReq[1].columnIndices + assertItem.catalogFilterReq[1] ); }); @@ -458,11 +398,11 @@ describe('Test for general CATALOG related messages:', () => { let StringFilteredResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 8) Request CatalogFilter: Filter(string) & check CatalogFilterResponse | `, async () => { - StringFilteredResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[2]); + StringFilteredResponse = await requestCatalogFilter(assertItem.catalogFilterReq[2]); assertCatalogFilterResponse( StringFilteredResponse, assertItem.catalogFilterResponse[2], - assertItem.catalogFilterReq[2].columnIndices + assertItem.catalogFilterReq[2] ); }); @@ -482,11 +422,11 @@ describe('Test for general CATALOG related messages:', () => { let SortedAndFilteredResponse: CARTA.ICatalogFilterResponse[]; test(`(Step 9) Request CatalogFilter: Sorting when Filter(string+number) is applied & check CatalogFilterResponse | `, async () => { - SortedAndFilteredResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[3]); + SortedAndFilteredResponse = await requestCatalogFilter(assertItem.catalogFilterReq[3]); assertCatalogFilterResponse( SortedAndFilteredResponse, assertItem.catalogFilterResponse[3], - assertItem.catalogFilterReq[3].columnIndices + assertItem.catalogFilterReq[3] ); }); diff --git a/src/test/CATALOG_LARGE.test.ts b/src/test/CATALOG_LARGE.test.ts index d3c92f45..781f9f24 100644 --- a/src/test/CATALOG_LARGE.test.ts +++ b/src/test/CATALOG_LARGE.test.ts @@ -1,5 +1,11 @@ import { CARTA } from 'carta-protobuf'; -import { Stream } from './MyClient'; +import { + Stream, + columnSlice, + assertCatalogFilterResponse, + requestCatalogFilter, + ICatalogFilterResponseExt, +} from './MyClient'; import { MessageController, ConnectionStatus } from './MessageController'; import config from './config.json'; @@ -18,76 +24,6 @@ interface IOpenCatalogFileAckExt extends CARTA.IOpenCatalogFileAck { lengthOfHeaders: number; } -interface ICatalogFilterResponseExt extends CARTA.ICatalogFilterResponse { - lengthOfColumns: number; - numberOfResponses: number; -} - -// ColumnData carries every type other than String as binary, so a row of a column is a -// fixed number of bytes rather than an element of an array. -const bytesPerElement = new Map([ - [CARTA.ColumnType.Uint8, 1], - [CARTA.ColumnType.Int8, 1], - [CARTA.ColumnType.Bool, 1], - [CARTA.ColumnType.Uint16, 2], - [CARTA.ColumnType.Int16, 2], - [CARTA.ColumnType.Uint32, 4], - [CARTA.ColumnType.Int32, 4], - [CARTA.ColumnType.Float, 4], - [CARTA.ColumnType.Uint64, 8], - [CARTA.ColumnType.Int64, 8], - [CARTA.ColumnType.Double, 8], -]); - -// Rows are compared as raw payload, which works for every column type without having to -// decode it. ProtobufProcessing cannot be used here because it needs the CARTACompute -// WASM global for the 64 bit types. -function columnSlice(column: CARTA.IColumnData, startRow: number, rowCount: number): (string | number)[] { - if (column.dataType === CARTA.ColumnType.String) { - return column.stringData!.slice(startRow, startRow + rowCount); - } - const elementSize = bytesPerElement.get(column.dataType!)!; - return Array.from(column.binaryData!.slice(startRow * elementSize, (startRow + rowCount) * elementSize)); -} - -function columnRowCount(column: CARTA.IColumnData): number { - if (column.dataType === CARTA.ColumnType.String) { - return column.stringData!.length; - } - return column.binaryData!.length / bytesPerElement.get(column.dataType!)!; -} - -function assertCatalogFilterResponse( - responses: CARTA.ICatalogFilterResponse[], - expected: ICatalogFilterResponseExt, - request: CARTA.ICatalogFilterRequest -) { - expect(responses.length).toEqual(expected.numberOfResponses); - const lastResponse = responses.slice(-1)[0]; - expect(lastResponse.fileId).toEqual(expected.fileId); - expect(Object.keys(lastResponse.columns).length).toEqual(expected.lengthOfColumns); - expect(Object.keys(lastResponse.columns)).toEqual(request.columnIndices.map((columnIndex) => `${columnIndex}`)); - expect(lastResponse.subsetDataSize).toEqual(expected.subsetDataSize); - expect(lastResponse.subsetEndIndex).toEqual(expected.subsetEndIndex); - expect(lastResponse.filterDataSize).toEqual(expected.filterDataSize); - expect(lastResponse.requestEndIndex).toEqual(expected.requestEndIndex); - expect(lastResponse.progress).toEqual(expected.progress); - Object.keys(lastResponse.columns).forEach((key) => { - expect(columnRowCount(lastResponse.columns[key])).toEqual(expected.subsetDataSize); - }); -} - -function requestCatalogFilter( - msgController: MessageController, - filterRequest: CARTA.ICatalogFilterRequest -): Promise { - // The stream has to be subscribed before the request is sent, otherwise the first - // responses are dropped. - const catalogFilterStream = Stream(CARTA.CatalogFilterResponse); - msgController.setCatalogFilterRequest(filterRequest); - return catalogFilterStream; -} - interface AssertItem { registerViewer: CARTA.IRegisterViewer; filelist: CARTA.IFileListRequest; @@ -362,7 +298,7 @@ describe('Test for large-size CATALOG: load whole table at one time', () => { test( `(Step 6) Request CatalogFilter: progress & check CatalogFilterResponse | `, async () => { - WholeTableResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[0]); + WholeTableResponse = await requestCatalogFilter(assertItem.catalogFilterReq[0]); wholeTableFirstChunk = WholeTableResponse[0]; console.log( `"${assertItem.catalogFileInfoReq.name}" CatalogFilterResponse progress :`, @@ -483,7 +419,7 @@ describe('Test for large-size CATALOG: Progressive load of rows', () => { test( `(Step 6-${i - 1}) Request CatalogFilter: subsetStartIndex of ${assertItem.catalogFilterReq[i].subsetStartIndex} & check CatalogFilterResponse | `, async () => { - const CatalogFilterResponse = await requestCatalogFilter(msgController, assertItem.catalogFilterReq[i]); + const CatalogFilterResponse = await requestCatalogFilter(assertItem.catalogFilterReq[i]); ProgressiveWindows[i] = CatalogFilterResponse[0]; assertCatalogFilterResponse( CatalogFilterResponse, diff --git a/src/test/MyClient.ts b/src/test/MyClient.ts index a295366e..c9068b8f 100644 --- a/src/test/MyClient.ts +++ b/src/test/MyClient.ts @@ -177,4 +177,96 @@ function ChannelMapStream(rasterTileDataLen: number, channels: number) { }); } -export { checkConnection, Stream, ChannelMapStream }; +// The expected values of a CATALOG_FILTER_RESPONSE which the protobuf message has no field +// for: how many messages the subset is streamed as, and how many columns each one carries. +interface ICatalogFilterResponseExt extends CARTA.ICatalogFilterResponse { + lengthOfColumns: number; + numberOfResponses: number; +} + +// ColumnData carries every type other than String as binary, so a row of a column is a +// fixed number of bytes rather than an element of an array. +const bytesPerElement = new Map([ + [CARTA.ColumnType.Uint8, 1], + [CARTA.ColumnType.Int8, 1], + [CARTA.ColumnType.Bool, 1], + [CARTA.ColumnType.Uint16, 2], + [CARTA.ColumnType.Int16, 2], + [CARTA.ColumnType.Uint32, 4], + [CARTA.ColumnType.Int32, 4], + [CARTA.ColumnType.Float, 4], + [CARTA.ColumnType.Uint64, 8], + [CARTA.ColumnType.Int64, 8], + [CARTA.ColumnType.Double, 8], +]); + +function columnRowCount(column: CARTA.IColumnData): number { + if (column.dataType === CARTA.ColumnType.String) { + return column.stringData!.length; + } + return column.binaryData!.length / bytesPerElement.get(column.dataType!)!; +} + +// Rows are compared as raw payload, which works for every column type without having to +// decode it. ProtobufProcessing cannot be used here because it needs the CARTACompute +// WASM global for the 64 bit types. +function columnSlice(column: CARTA.IColumnData, startRow: number, rowCount: number): (string | number)[] { + if (column.dataType === CARTA.ColumnType.String) { + return column.stringData!.slice(startRow, startRow + rowCount); + } + const elementSize = bytesPerElement.get(column.dataType!)!; + return Array.from(column.binaryData!.slice(startRow * elementSize, (startRow + rowCount) * elementSize)); +} + +function stringColumn(column: CARTA.IColumnData): string[] { + expect(column.dataType).toEqual(CARTA.ColumnType.String); + return column.stringData!; +} + +function doubleColumn(column: CARTA.IColumnData): number[] { + expect(column.dataType).toEqual(CARTA.ColumnType.Double); + return Array.from(new Float64Array(column.binaryData!.slice().buffer)); +} + +// Assert the last message of a streamed subset against the request which asked for it +function assertCatalogFilterResponse( + responses: CARTA.ICatalogFilterResponse[], + expected: ICatalogFilterResponseExt, + request: CARTA.ICatalogFilterRequest +) { + expect(responses.length).toEqual(expected.numberOfResponses); + const lastResponse = responses.slice(-1)[0]; + expect(lastResponse.fileId).toEqual(expected.fileId); + expect(Object.keys(lastResponse.columns).length).toEqual(expected.lengthOfColumns); + expect(Object.keys(lastResponse.columns)).toEqual(request.columnIndices.map((columnIndex) => `${columnIndex}`)); + expect(lastResponse.subsetDataSize).toEqual(expected.subsetDataSize); + expect(lastResponse.subsetEndIndex).toEqual(expected.subsetEndIndex); + expect(lastResponse.filterDataSize).toEqual(expected.filterDataSize); + expect(lastResponse.requestEndIndex).toEqual(expected.requestEndIndex); + expect(lastResponse.progress).toEqual(expected.progress); + Object.keys(lastResponse.columns).forEach((key) => { + expect(columnRowCount(lastResponse.columns[key])).toEqual(expected.subsetDataSize); + }); +} + +function requestCatalogFilter(filterRequest: CARTA.ICatalogFilterRequest): Promise { + const msgController = MessageController.Instance; + // The stream has to be subscribed before the request is sent, otherwise the first + // responses are dropped. + const catalogFilterStream = Stream(CARTA.CatalogFilterResponse); + msgController.setCatalogFilterRequest(filterRequest); + return catalogFilterStream; +} + +export { + checkConnection, + Stream, + ChannelMapStream, + columnRowCount, + columnSlice, + stringColumn, + doubleColumn, + assertCatalogFilterResponse, + requestCatalogFilter, +}; +export type { ICatalogFilterResponseExt };