From 3e7fb3d3069540949efda01dac4c9b0d99a09f7e Mon Sep 17 00:00:00 2001 From: Ollie Tooth Date: Tue, 30 Jun 2026 15:20:18 +0100 Subject: [PATCH 1/4] Add support for monthly climatology calculation to DataLoader ABC. --- OceanOSSE/io/dataloader.py | 103 ++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 18 deletions(-) diff --git a/OceanOSSE/io/dataloader.py b/OceanOSSE/io/dataloader.py index 8e6b721..3585e0c 100644 --- a/OceanOSSE/io/dataloader.py +++ b/OceanOSSE/io/dataloader.py @@ -14,6 +14,7 @@ import logging from typing import Self +import numpy as np import xarray as xr logger = logging.getLogger(__name__) @@ -42,6 +43,9 @@ class DataLoader(abc.ABC): Mapping of standard dimension names to input dataset dimension names. _coordinates : dict[str, str] Mapping of standard coordinate names to input dataset coordinate names. + _table : str + Name of the table in the .toml configuration file. Options are 'inputs' + or 'climatology'. Default is 'inputs'. """ def __init__( @@ -49,6 +53,7 @@ def __init__( source: dict[str, dict], dimensions: dict[str, str], coordinates: dict[str, str], + table: str = "inputs" ): # -- Verify Inputs -- # if not isinstance(source, dict): @@ -57,25 +62,29 @@ def __init__( raise TypeError("``dimensions`` must be a specfied as a dictionary.") if not isinstance(coordinates, dict): raise TypeError("``coordinates`` must be a specfied as a dictionary.") + if not isinstance(table, str): + raise TypeError("``table`` must be a specfied as a string.") # -- Class Attributes -- # self._source = source self._dimensions = dimensions self._coordinates = coordinates + self._table = table def __repr__(self) -> str: return ( f"{type(self).__name__}(" f"source={self._source!r}, " f"dimensions={self._dimensions!r}, " - f"coordinates={self._coordinates!r})" + f"coordinates={self._coordinates!r}, " + f"table={self._table!r})" ) @classmethod @abc.abstractmethod - def from_config(cls, config: dict) -> Self: + def from_config(cls, config: dict, table: str = "inputs") -> Self: """ - Abstract class method to instantiate a DataLoader from the `[inputs]` table + Abstract class method to instantiate a DataLoader from the specified table of the .toml configuration file. This is the required constructor for all DataLoader subclasses - plugin @@ -86,6 +95,9 @@ def from_config(cls, config: dict) -> Self: config : dict Configuration dictionary containing input parameters from .toml configuration file. + table : str + Name of the table in the .toml configuration file. Options are + 'inputs' or 'climatology'. Default is 'inputs'. Returns ------- @@ -108,6 +120,35 @@ def load_data(self) -> xr.Dataset: """ ... + def compute_monthly_climatology(self) -> xr.Dataset: + """ + Compute monthly climatology from the input xarray.Dataset. + + Returns + ------- + xarray.Dataset + Dataset containing monthly climatology of the input variables. + """ + # -- Load Input Dataset -- # + ds = self.load_data() + start_yr = ds['time'].dt.year.min().item() + end_yr = ds['time'].dt.year.max().item() + + # -- Compute Monthly Climatology -- # + ds_clim = ds.groupby("time.month").mean(dim="time", skipna=True) + + # -- Assign Time Bounds -- # + # Update time bounds to reflect climatological period: + ds_clim['time_bnds'] = xr.DataArray( + np.zeros((12, 2), dtype='datetime64[ns]'), + dims=('month', 'bnds'), + coords={'month': ds_clim['month']}, + ) + ds_clim['time_bnds'].data[:, 0] = (np.datetime64(f'{start_yr}-01', 'M') + (np.timedelta64(1, 'M') * np.arange(ds['time'].size))).astype('datetime64[ns]') + ds_clim['time_bnds'].data[:, 1] = (np.datetime64(f'{end_yr}-01', 'M') + (np.timedelta64(1, 'M') * np.arange(ds['time'].size))).astype('datetime64[ns]') + + return ds_clim + def _standardise_dataset(self, ds: xr.Dataset) -> xr.Dataset: """ Standardise gridded ocean model xarray.Dataset by renaming dimensions and coordinates to OceanOSSE standard names. @@ -141,7 +182,7 @@ def _validate_dataset(self, ds: xr.Dataset) -> None: Parameters ---------- - dataset : xarray.Dataset + ds : xarray.Dataset Dataset containing standardised gridded ocean model variables. Raises @@ -150,7 +191,10 @@ def _validate_dataset(self, ds: xr.Dataset) -> None: If the dataset does not contain the required variables or dimensions. """ # -- Validate Required Dimensions -- # - required_dims = ["time", "lev", "j", "i"] + if self._table == "climatology": + required_dims = ["month", "lev", "j", "i"] + else: + required_dims = ["time", "lev", "j", "i"] missing_dims = [dim for dim in required_dims if dim not in ds.dims] if missing_dims: raise ValueError( @@ -159,7 +203,10 @@ def _validate_dataset(self, ds: xr.Dataset) -> None: ) # -- Validate Required Coordinates -- # - required_coords = ["time", "depth", "lat", "lon"] + if self._table == "climatology": + required_coords = ["month", "depth", "lat", "lon"] + else: + required_coords = ["time", "depth", "lat", "lon"] missing_coords = [coord for coord in required_coords if coord not in ds.coords] if missing_coords: raise ValueError( @@ -188,37 +235,57 @@ def __init__( source: dict[str, dict], dimensions: dict[str, str], coordinates: dict[str, str], + table: str = "inputs", ) -> None: # -- Initialise parent DataLoader class -- # - super().__init__(source, dimensions, coordinates) + super().__init__(source, dimensions, coordinates, table) @classmethod - def from_config(cls, config: dict) -> Self: + def from_config(cls, config: dict, table: str = "inputs") -> Self: """ - Instantiate a NetCDFDataLoader from the `[inputs]` table of the .toml configuration file. + Instantiate a NetCDFDataLoader from the specified table of the .toml configuration file. + + Parameters + ---------- + config : dict + Configuration dictionary. + table : str, optional + Name of the table in the .toml configuration file. + Options are 'inputs' or 'climatology'. Default is 'inputs'. + + Returns + ------- + NetCDFDataLoader + Instantiated NetCDFDataLoader object. """ # -- Verify Input -- # if not isinstance(config, dict): raise TypeError("config must be a dictionary.") + if not isinstance(table, str): + raise TypeError("table must be a string.") + if table not in ["inputs", "climatology"]: + raise ValueError( + "table must be either 'inputs' or 'climatology'." + ) # -- Instantiate DataLoader with source dict from config -- # - source = config["inputs"].get("variables", None) + source = config[table].get("variables", None) if source is None: raise ValueError( - "Missing 'variables' entry in [inputs] table of config .toml file." + f"Missing 'variables' entry in [{table}] table of config .toml file." ) - dimensions = config["inputs"].get("dimensions", None) + dimensions = config[table].get("dimensions", None) if dimensions is None: raise ValueError( - "Missing 'dimensions' entry in [inputs] table of config .toml file." + f"Missing 'dimensions' entry in [{table}] table of config .toml file." ) - coordinates = config["inputs"].get("coordinates", None) + coordinates = config[table].get("coordinates", None) if coordinates is None: raise ValueError( - "Missing 'coordinates' entry in [inputs] table of config .toml file." + f"Missing 'coordinates' entry in [{table}] table of config .toml file." ) - return cls(source=source, dimensions=dimensions, coordinates=coordinates) + return cls(source=source, dimensions=dimensions, coordinates=coordinates, table=table) def _open_dataset( self, @@ -342,13 +409,13 @@ def load_data(self) -> xr.Dataset: ds = xr.merge(ds_list, compat="no_conflicts", join="exact") # -- Standardise dataset dimensions and coordinates -- # - ds = self._standardise_dataset(ds) + ds = self._standardise_dataset(ds=ds) logging.info( f"--> Completed: Standardised dataset dimensions {list(ds.sizes.keys())} and coordinates {list(ds.coords.keys())}." ) # -- Validate standardised dataset -- # - self._validate_dataset(ds) + self._validate_dataset(ds=ds) logging.info("--> Completed: Validated standardised dataset.") return ds From 23aefcefc25ca0db80edf6624b84614388822daf Mon Sep 17 00:00:00 2001 From: Ollie Tooth Date: Tue, 30 Jun 2026 15:21:09 +0100 Subject: [PATCH 2/4] Add support to write monthly climatology using DataWriter + NetCDFDataWriter. --- OceanOSSE/io/datawriter.py | 106 ++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 30 deletions(-) diff --git a/OceanOSSE/io/datawriter.py b/OceanOSSE/io/datawriter.py index 1b0a1f4..6293a3c 100644 --- a/OceanOSSE/io/datawriter.py +++ b/OceanOSSE/io/datawriter.py @@ -74,6 +74,7 @@ def __init__( date_format: str, chunks: dict[str, int] | None = None, writer_kwargs: dict[str, any] | None = None, + climatology: bool = False, ) -> None: # -- Validate Input -- # if not isinstance(dimensions, dict): @@ -90,6 +91,8 @@ def __init__( raise TypeError("``chunks`` must be a dict or None.") if (writer_kwargs is not None) and not isinstance(writer_kwargs, dict): raise TypeError("``writer_kwargs`` must be a dict or None.") + if not isinstance(climatology, bool): + raise TypeError("``climatology`` must be a boolean.") # -- Class Attributes -- # self._dimensions = dimensions @@ -99,6 +102,7 @@ def __init__( self._date_format = date_format self._chunks = chunks self._writer_kwargs = writer_kwargs + self._climatology = climatology def __repr__(self) -> str: return ( @@ -107,12 +111,13 @@ def __repr__(self) -> str: f"output_name={self._output_name!r}, " f"date_format={self._date_format!r}, " f"chunks={self._chunks!r}, " - f"writer_kwargs={self._writer_kwargs!r})" + f"writer_kwargs={self._writer_kwargs!r}, " + f"climatology={self._climatology!r})" ) @classmethod @abc.abstractmethod - def from_config(cls, config: dict) -> Self: + def from_config(cls, config: dict, climatology: bool = False) -> Self: """ Abstract class method to instantiate a DataWriter from the `[outputs]` table of the .toml configuration file. @@ -125,6 +130,8 @@ def from_config(cls, config: dict) -> Self: config : dict Configuration dictionary containing input parameters from .toml configuration file. + climatology : bool, optional + Whether the dataset is a climatology. Default is False. Returns ------- @@ -182,6 +189,7 @@ def _get_output_filepath( output_name: str, file_format: str, date_format: str, + climatology: bool = False, ) -> str: """ Define resolved filepath to OceanOSSE output file(s). @@ -199,6 +207,9 @@ def _get_output_filepath( date_format : str Date format for datetime limits in output filename. Options are 'Y' (YYYY), 'M' (YYYY-MM) or 'D' (YYYY-MM-DD). + climatology : bool, optional + If True, the output filename will include the 'climatology' + time bounds. Default is False. """ # -- Validate Inputs -- # if not isinstance(ds, xr.Dataset): @@ -209,32 +220,37 @@ def _get_output_filepath( raise TypeError("output_name must be a string.") if file_format not in ["netcdf", "zarr"]: raise ValueError("file_format must be either 'netcdf' or 'zarr'.") + if not isinstance(climatology, bool): + raise ValueError("climatology must be a boolean.") # -- Create Date String for Output Fileapath -- # # Define time-limits of output dataset: - time_limits = ds["time"].values[[0, -1]] - - # Create date string from CFTime datetime objects: - if isinstance(time_limits[0], cftime.datetime): - if date_format == "Y": - fmt = "%Y" - elif date_format == "M": - fmt = "%Y-%m" - elif date_format == "D": - fmt = "%Y-%m-%d" + if climatology: + date_str = f"{np.datetime_as_string(ds['time_bnds'][0, 0], unit='M')}_{np.datetime_as_string(ds['time_bnds'][-1, -1], unit='M')}_climatology" + else: + time_limits = ds["time"].values[[0, -1]] + + # Create date string from CFTime datetime objects: + if isinstance(time_limits[0], cftime.datetime): + if date_format == "Y": + fmt = "%Y" + elif date_format == "M": + fmt = "%Y-%m" + elif date_format == "D": + fmt = "%Y-%m-%d" + else: + raise ValueError( + f"Invalid date_format: '{date_format}'. Options are 'Y', 'M', 'D'." + ) + date_str = f"{time_limits[0].strftime(fmt)}-{time_limits[1].strftime(fmt)}" + + # Create date string from numpy datetime64: + elif isinstance(time_limits[0], np.datetime64): + date_str = f"{np.datetime_as_string(time_limits[0], unit=date_format)}-{np.datetime_as_string(time_limits[1], unit=date_format)}" else: - raise ValueError( - f"Invalid date_format: '{date_format}'. Options are 'Y', 'M', 'D'." + raise TypeError( + f"Invalid type ({type(time_limits[0])}) for dates. Expected cftime.datetime or np.datetime64." ) - date_str = f"{time_limits[0].strftime(fmt)}-{time_limits[1].strftime(fmt)}" - - # Create date string from numpy datetime64: - elif isinstance(time_limits[0], np.datetime64): - date_str = f"{np.datetime_as_string(time_limits[0], unit=date_format)}-{np.datetime_as_string(time_limits[1], unit=date_format)}" - else: - raise TypeError( - f"Invalid type ({type(time_limits[0])}) for dates. Expected cftime.datetime or np.datetime64." - ) # -- Define Output Filepath -- # if file_format == "netcdf": @@ -268,6 +284,8 @@ class NetCDFDataWriter(DataWriter): Default is None, meaning no chunking is applied. writer_kwargs : dict[str, any], optional Additional keyword arguments to pass to xarray.Dataset.to_netcdf. + climatology : bool, optional + Whether the dataset is a climatology. Default is False. """ def __init__( @@ -279,6 +297,7 @@ def __init__( date_format: str, chunks: dict[str, int] | None = None, writer_kwargs: dict[str, any] | None = None, + climatology: bool = False, ) -> None: # -- Initialise parent DataWriter class -- # super().__init__( @@ -289,12 +308,20 @@ def __init__( date_format=date_format, chunks=chunks, writer_kwargs=writer_kwargs, + climatology=climatology ) @classmethod - def from_config(cls, config: dict) -> Self: + def from_config(cls, config: dict, climatology: bool = False) -> Self: """ Instantiate a NetCDFDataWriter from the `[outputs]` table of the .toml configuration file. + + Parameters + ---------- + config : dict + Configuration dictionary. + climatology : bool, optional + Whether the dataset is a climatology. Default is False. """ # -- Verify Input -- # if not isinstance(config, dict): @@ -311,9 +338,10 @@ def from_config(cls, config: dict) -> Self: date_format=outputs["date_format"], chunks=outputs.get("chunks", None), writer_kwargs=outputs.get("writer_kwargs", None), + climatology=climatology ) - def write_data(self, ds: xr.Dataset) -> None: + def write_data(self, ds: xr.Dataset) -> str: """ Write OceanOSSE output xarray.Dataset to a netCDF file. @@ -341,23 +369,41 @@ def write_data(self, ds: xr.Dataset) -> None: output_name=self._output_name, file_format="netcdf", date_format=self._date_format, + climatology=self._climatology ) # -- Reconstruct Dataset with Original Dimension and Coordinate Names -- # - ds = self._reconstruct_dataset(ds) + if not self._climatology: + ds = self._reconstruct_dataset(ds) - # -- Optionally Apply Chunking -- # - if self._chunks is not None: - ds = ds.chunk(self._chunks) + # -- Apply Optional Chunking -- # + if self._climatology: + ds = ds.chunk({"month": 1}) + else: + if self._chunks is not None: + ds = ds.chunk(self._chunks) # -- Write Dataset to NetCDF -- # if self._writer_kwargs is None: + if not self._climatology: + unlimited_dim = self._dimensions.get("time") + else: + unlimited_dim = self._dimensions.get("month") + # Default writer_kwargs for netCDF output: self._writer_kwargs = { - "unlimited_dims": self._dimensions.get("time"), + "unlimited_dims": unlimited_dim, "mode": "w", } + else: + if self._climatology: + self._writer_kwargs["unlimited_dims"] = self._dimensions.get("month") + + # Remove existing encoding: + ds.encoding.clear() ds.to_netcdf(path=output_filepath, **self._writer_kwargs) logging.info( f"--> Completed: Written output dataset to netCDF file: {output_filepath}" ) + + return output_filepath From 29bef60e647ff8f2cae792ed0a387e52342c970e Mon Sep 17 00:00:00 2001 From: Ollie Tooth Date: Tue, 30 Jun 2026 15:22:12 +0100 Subject: [PATCH 3/4] Add support for calculating monthly climatology from input dataset in OceanOSSE run_pipeline() driver. --- OceanOSSE/pipeline.py | 59 +++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/OceanOSSE/pipeline.py b/OceanOSSE/pipeline.py index e766428..58d086b 100644 --- a/OceanOSSE/pipeline.py +++ b/OceanOSSE/pipeline.py @@ -28,9 +28,9 @@ # -- Factory Functions -- # -def _create_DataLoader(config: dict) -> DataLoader: +def _create_DataLoader(config: dict, table: str = "inputs") -> DataLoader: """ - Instantiate a DataLoader from the `[inputs]` table of the .toml configuration file. + Instantiate a DataLoader from the specified table of the .toml configuration file. Options: - Built-in Registry: @@ -42,24 +42,28 @@ def _create_DataLoader(config: dict) -> DataLoader: # -- Validate Inputs -- # if not isinstance(config, dict): raise TypeError("``config`` must be a dictionary.") + if not isinstance(table, str): + raise TypeError("``table`` must be a string.") + if table not in ["inputs", "climatology"]: + raise ValueError("``table`` must be either 'inputs' or 'climatology'.") # -- Instantiate DataLoader -- # - inputs = config["inputs"] + table = config[table] # 1. Plugin DataLoader: - if (inputs.get("module") is not None) and (inputs.get("name") is not None): + if (table.get("module") is not None) and (table.get("name") is not None): # -- Import custom DataLoader class -- # data_loader = import_class( - module=inputs["module"], class_name=inputs["name"], class_type=DataLoader + module=table["module"], class_name=table["name"], class_type=DataLoader ) logger.info( - f"Completed: Created DataLoader from Plugin: {inputs['module']}.{inputs['name']}" + f"Completed: Created DataLoader from Plugin: {table['module']}.{table['name']}" ) # 2. Registry DataLoader: else: # -- Use DataLoader class from registry -- # - format = inputs.get("format", "netcdf4") + format = table.get("format", "netcdf4") try: data_loader = _DATA_LOADER_REGISTRY[format] except KeyError as e: @@ -70,7 +74,7 @@ def _create_DataLoader(config: dict) -> DataLoader: f"Completed: Created DataLoader from Registry -> {format}: {data_loader.__name__}" ) - return data_loader.from_config(config=config) + return data_loader.from_config(config=config, table=table) def _create_ObsSampler(config: dict) -> ObsSampler: @@ -167,7 +171,7 @@ def _create_Regridder(config: dict) -> Regridder: return regridder.from_config(config=config) -def _create_DataWriter(config: dict) -> DataWriter: +def _create_DataWriter(config: dict, climatology: bool = False) -> DataWriter: """ Instantiate a DataWriter from the `[outputs]` table of the .toml configuration file. @@ -181,6 +185,8 @@ def _create_DataWriter(config: dict) -> DataWriter: # -- Validate Inputs -- # if not isinstance(config, dict): raise TypeError("``config`` must be a dictionary.") + if not isinstance(climatology, bool): + raise TypeError("``climatology`` must be a boolean.") # -- Instantiate DataWriter -- # outputs = config["outputs"] @@ -209,7 +215,7 @@ def _create_DataWriter(config: dict) -> DataWriter: f"Completed: Created DataWriter from Registry -> {format}: {data_writer.__name__}" ) - return data_writer.from_config(config=config) + return data_writer.from_config(config=config, climatology=climatology) # -- Define Pipeline Functions -- # @@ -218,7 +224,8 @@ def run_pipeline(args: dict) -> None: Run OceanOSSE pipeline using specified config .ini file. Pipeline Steps: - 1. Instantiate DataLoader -> Load standardised ocean model dataset. + 1a. Instantiate DataLoader -> Load standardised ocean model dataset. + 1b. Compute or Load monthly climatology dataset. 2. Instantiate ObsSampler -> Sample synthetic ocean observations from model dataset. 3. Instantiate Regridder -> Regrid synthetic observations onto original model grid. 4. Instantiate DataWriter -> Write output dataset to file. @@ -235,9 +242,32 @@ def run_pipeline(args: dict) -> None: logger.info(f"Completed: Read & validated config file -> {args['config_file']}") # Load ocean model dataset using DataLoader: - data_loader = _create_DataLoader(config=config) - logger.info(f"In Progress: Loading ocean model dataset using {data_loader}...") + data_loader = _create_DataLoader(config=config, table="inputs") + logger.info(f"In Progress: Loading ocean input model dataset using {data_loader}...") ds_mdl = data_loader.load_data() + logger.info("Completed: Loaded ocean model input dataset.") + + if config["climatology"].get("read_climatology", False): + # Calculate monthly climatology from model dataset: + logger.info( + "In Progress: Calculating monthly climatology from model input dataset..." + ) + ds_clim = data_loader.compute_monthly_climatology() + logger.info("Completed: Calculated monthly climatology from model input dataset.") + clim_writer = _create_DataWriter(config=config, climatology=True) + logger.info( + f"In Progress: Writing monthly climatology to file using {clim_writer}..." + ) + clim_output_filepath = clim_writer.write_data(ds=ds_clim) + logger.info(f"Completed: Written monthly climatology to file: {clim_output_filepath}") + else: + # Load monthly climatology from file: + clim_loader = _create_DataLoader(config=config, table="climatology") + logger.info( + f"In Progress: Loading monthly climatology from file using {clim_loader}..." + ) + ds_clim = clim_loader.load_data() + logger.info("Completed: Loaded monthly climatology from file.") # === Sampling === # logger.info("==== Sampling ====") @@ -262,7 +292,8 @@ def run_pipeline(args: dict) -> None: # Write output dataset to file using DataWriter: data_writer = _create_DataWriter(config=config) logger.info(f"In Progress: Writing output dataset to file using {data_writer}...") - data_writer.write_data(ds=ds_regridded) + output_filepath = data_writer.write_data(ds=ds_regridded) + logger.info(f"Completed: Written output dataset to file: {output_filepath}") # Close all files: for ds in [ds_mdl, ds_obs, ds_regridded]: ds.close() From 59508df75daa7154718a3eff2ef11c99d91ee492 Mon Sep 17 00:00:00 2001 From: Ollie Tooth Date: Tue, 30 Jun 2026 15:23:23 +0100 Subject: [PATCH 4/4] Add integration tests for DataLoader climatology calculation. Add new pixi task to run integration tests on Anemone HPC. --- pixi.toml | 3 +- tests/conftest.py | 24 ++++++++++++-- tests/integration/test_example.py | 52 ++++++++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/pixi.toml b/pixi.toml index c51e107..791b82b 100644 --- a/pixi.toml +++ b/pixi.toml @@ -65,7 +65,8 @@ twine = "*" # === Tasks === # [feature.dev.tasks] -tests = { cmd = "pytest --maxfail=1 --tb=short tests/unit/", description = "Run OceanOSSE unit tests using pytest." } +unit_tests = { cmd = "pytest --maxfail=1 --tb=short tests/unit/", description = "Run OceanOSSE unit tests using pytest." } +integration_tests = { cmd = "pytest --maxfail=1 --tb=short tests/integration/", description = "Run OceanOSSE integration tests using pytest. For use on NOC Anemone HPC." } lint = { cmd = "ruff check OceanOSSE/ tests/", description = "Run code linting using Ruff." } fix = { cmd = "ruff check OceanOSSE/ tests/ --fix", description = "Run code linting and auto-fix issues using Ruff." } format = { cmd = "ruff format OceanOSSE/ tests/", description = "Run code formatting using Ruff." } diff --git a/tests/conftest.py b/tests/conftest.py index ef1dd5f..3d19536 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,11 +30,31 @@ def example_config() -> dict: }, "variables": { "thetao_con": { - "path": "/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1/eORCA1_ERA5_1m_grid_T_202312-202312.nc", + "path": "/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1/eORCA1_ERA5_1m_grid_T_2023??-2023??.nc", "open_kwargs": {"engine": "netcdf4"}, }, "so_abs": { - "path": "/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1/eORCA1_ERA5_1m_grid_T_202312-202312.nc", + "path": "/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1/eORCA1_ERA5_1m_grid_T_2023??-2023??.nc", + "open_kwargs": {"engine": "netcdf4"}, + }, + }, + }, + "climatology": { + "read_climatology": False, + "dimensions": {"month": "time_counter", "lev": "deptht", "j": "y", "i": "x"}, + "coordinates": { + "lat": "nav_lat", + "lon": "nav_lon", + "depth": "deptht", + "month": "time_counter", + }, + "variables": { + "thetao_con": { + "path": "/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1/eORCA1_ERA5_1m_grid_T_2023??-2023??.nc", + "open_kwargs": {"engine": "netcdf4"}, + }, + "so_abs": { + "path": "/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1/eORCA1_ERA5_1m_grid_T_2023??-2023??.nc", "open_kwargs": {"engine": "netcdf4"}, }, }, diff --git a/tests/integration/test_example.py b/tests/integration/test_example.py index ced2883..61c1821 100644 --- a/tests/integration/test_example.py +++ b/tests/integration/test_example.py @@ -19,17 +19,27 @@ # Example integration test class: class TestOceanOSSEExample: - def test_DataLoader_config(self, example_config: dict): + def test_DataLoader_inputs_config(self, example_config: dict): # Initialise example DataLoader using test configuration: - dataloader = NetCDFDataLoader.from_config(config=example_config) + dataloader = NetCDFDataLoader.from_config(config=example_config, table="inputs") # Verify DataLoader attributes: assert dataloader._source == example_config["inputs"]["variables"] assert dataloader._dimensions == example_config["inputs"]["dimensions"] assert dataloader._coordinates == example_config["inputs"]["coordinates"] + assert dataloader._table == "inputs" - def test_DataLoader_load_data(self, example_config: dict): + def test_DataLoader_climatology_config(self, example_config: dict): # Initialise example DataLoader using test configuration: - dataloader = NetCDFDataLoader.from_config(config=example_config) + dataloader = NetCDFDataLoader.from_config(config=example_config, table="climatology") + # Verify DataLoader attributes: + assert dataloader._source == example_config["climatology"]["variables"] + assert dataloader._dimensions == example_config["climatology"]["dimensions"] + assert dataloader._coordinates == example_config["climatology"]["coordinates"] + assert dataloader._table == "climatology" + + def test_DataLoader_inputs_load_data(self, example_config: dict): + # Initialise example DataLoader using test configuration: + dataloader = NetCDFDataLoader.from_config(config=example_config, table="inputs") # Load data using DataLoader: ds = dataloader.load_data() # Verify the integrity of xarray.Dataset: @@ -44,6 +54,40 @@ def test_DataLoader_load_data(self, example_config: dict): example_config["inputs"]["coordinates"].keys() ) + def test_DataLoader_climatology_load_data(self, example_config: dict): + # Initialise example DataLoader using test configuration: + dataloader = NetCDFDataLoader.from_config(config=example_config, table="climatology") + # Load data using DataLoader: + ds = dataloader.load_data() + # Verify the integrity of xarray.Dataset: + assert isinstance(ds, xr.Dataset) + assert set(ds.data_vars.keys()) == set( + example_config["climatology"]["variables"].keys() + ) + assert set(ds.sizes.keys()) == set( + example_config["climatology"]["dimensions"].keys() + ) + assert set(ds.coords.keys()) == set( + example_config["climatology"]["coordinates"].keys() + ) + + def test_DataLoader_compute_monthly_climatology(self, example_config: dict): + # Initialise example DataLoader using test configuration: + dataloader = NetCDFDataLoader.from_config(config=example_config, table="inputs") + # Compute monthly climatology from DataLoader: + ds = dataloader.compute_monthly_climatology() + # Verify the integrity of xarray.Dataset: + assert isinstance(ds, xr.Dataset) + assert set(ds.data_vars.keys()) == set( + list(example_config["climatology"]["variables"].keys()) + ["time_bnds"] + ) + assert set(ds.sizes.keys()) == set( + list(example_config["climatology"]["dimensions"].keys()) + ["bnds"] + ) + assert set(ds.coords.keys()) == set( + example_config["climatology"]["coordinates"].keys() + ) + def test_Sampler_config(self, example_config: dict): # Initialise example Sampler using test configuration: sampler = MockObsSampler.from_config(config=example_config)