From 61ce383d35178043d08655c571ebf8dae5b8eba8 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 22 Jun 2026 09:44:34 -0700 Subject: [PATCH 01/33] Create a new folder for ISMIP7 postprocessing This is an exact copy of ISMIP6 postprocessing as a starting point --- .../create_mapfile_mali_to_ismip6.py | 104 +++ .../post_process_mali_to_ismip6.py | 237 ++++++ .../process_flux_variables.py | 518 ++++++++++++++ .../process_state_variables.py | 675 ++++++++++++++++++ .../recalculate_missing_2d_state_vars.py | 144 ++++ 5 files changed, 1678 insertions(+) create mode 100644 landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip6.py create mode 100755 landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip6.py create mode 100644 landice/output_processing_li/ismip7_postprocessing/process_flux_variables.py create mode 100755 landice/output_processing_li/ismip7_postprocessing/process_state_variables.py create mode 100755 landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py diff --git a/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip6.py b/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip6.py new file mode 100644 index 000000000..b9732b227 --- /dev/null +++ b/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip6.py @@ -0,0 +1,104 @@ +from mpas_tools.scrip.from_mpas import scrip_from_mpas +from subprocess import check_call +import os +import netCDF4 +import xarray as xr + +def build_mapping_file(mali_mesh_file, + mapping_file, res_ismip6_grid, + ismip6_grid_file=None, + method_remap=None): + """ + Build a mapping file if it does not exist. + Mapping file is then used to remap the MALI source file to the + ISMIP6 ppolarstero grid + + Parameters + ---------- + + mali_mesh_file : str + mali file + + mapping_file : str + weights for interpolation from mali_mesh_file to ismip6_grid_file + + res_ismip6_grid: str + resolution of the ismip6 grid in kilometers + + ismip6_grid_file : str, optional + The ISMIP6 file if mapping file does not exist + + method_remap : str, optional + Remapping method used in building a mapping file + """ + + if os.path.exists(mapping_file): + print(f"Mapping file exists. Not building a new one.") + return + + if ismip6_grid_file is None: + raise ValueError("Mapping file does not exist. To build one, ISMIP6 " + "grid file with '-f' should be provided. " + "Type --help for info") + + if method_remap is None: + method_remap = "conserve" + + ismip6_scripfile = f"temp_ismip6_{res_ismip6_grid}km_scrip.nc" + mali_scripfile = "temp_mali_scrip.nc" + ismip6_projection = "ais-bedmap2" + + # create the ismip6 scripfile if mapping file does not exist + # this is the projection of ismip6 data for Antarctica + print(f"Mapping file does not exist. Building one based on " + f"the input/ouptut meshes") + print(f"Creating temporary scripfiles " + f"for ismip6 grid and mali mesh...") + + # create a scripfile for ismip6 grid + args = ["create_SCRIP_file_from_planar_rectangular_grid.py", + "--input", ismip6_grid_file, + "--scrip", ismip6_scripfile, + "--proj", ismip6_projection, + "--rank", "2"] + + check_call(args) + + # create a MALI mesh scripfile + scrip_from_mpas(mali_mesh_file, mali_scripfile) + + # create a mapping file using ESMF weight gen + print(f"Creating a mapping file... " + f"Mapping method used: {method_remap}") + + # generate a mapping file using the scrip files + + # On compute node, ESMG regridder needs to be called with srun + # On head nodes or local machines it does not. + # Here, assuming a compute node has hostname starting with nid, + # which is the case on Cori. Modify as needed for other machines. + # Also assuming we can use multiple cores on a compute node. + hostname = os.uname()[1] + if hostname.startswith('nid'): + args = (["srun", "-n", "12", "ESMF_RegridWeightGen", + "-s", mali_scripfile, + "-d", ismip6_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) + else: + args = (["ESMF_RegridWeightGen", + "-s", mali_scripfile, + "-d", ismip6_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) + + check_call(args) + + # remove the temporary scripfiles once the mapping file is generated + print(f"Removing the temporary mesh and scripfiles...") + os.remove(ismip6_scripfile) + os.remove(mali_scripfile) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip6.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip6.py new file mode 100755 index 000000000..7bf8eb7fd --- /dev/null +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip6.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python + +""" +This script processes MALI simulation outputs (both state and flux) +in the required format by the ISMIP6 experimental protocol. +The input state files (i.e., output files from MALI) need to have been +concatenated to have yearly data, which can be done using 'ncrcat' command +before using this script. +""" + +import argparse +from subprocess import check_call +import os +import shutil +import numpy as np +from netCDF4 import Dataset +from create_mapfile_mali_to_ismip6 import build_mapping_file +from process_state_variables import generate_output_2d_state_vars, \ + process_state_vars, generate_output_1d_vars +from process_flux_variables import generate_output_2d_flux_vars, \ + do_time_avg_flux_vars, clean_flux_fields_before_time_averaging + + +def main(): + parser = argparse.ArgumentParser( + description='process MALI outputs for the ISMIP6' + 'submission') + parser.add_argument("-e", "--exp_name", dest="exp", + required=True, + help="ISMIP6 experiment name (e.g., exp05") + parser.add_argument("-i_state", "--input_state", dest="input_file_state", + required=False, help="mpas output state variables") + parser.add_argument("-i_flux", "--input_flux", dest="input_file_flux", + required=False, help="mpas output flux variables") + parser.add_argument("-i_mesh", "--input_mesh", dest="input_file_grid", + required=False, help="MALI file with mesh information") + parser.add_argument("-g", "--global_stats_file", dest="global_stats_file", + required=False, help="globalStats.nc file") + parser.add_argument("-p", "--output_path", dest="output_path", + required=False, + help="path to which the final output files" + " will be saved") + parser.add_argument("--mali_mesh_name", dest="mali_mesh_name", + required=True, + help="mali mesh name (e.g., AIS_8to30km)") + parser.add_argument("--mapping_file", dest="mapping_file", + required=False, + help="mapping file name from MALI mesh to ISMIP6 grid") + parser.add_argument("--ismip6_grid_file", dest="ismip6_grid_file", + required=True, + help="Input ismip6 mesh file.") + parser.add_argument("--method", dest="method_remap", default="conserve", + required=False, + help="mapping method. Default='conserve'") + parser.add_argument("--res", dest="res_ismip6_grid", + required=True, + help="resolution of the ismip6 grid, (e.g. 8 for 8km res)") + args = parser.parse_args() + + print("\n---Checking the coordinate variables of the ismip6 grid file---") + data_ismip6 = Dataset(args.ismip6_grid_file, "r") + if 'x' and 'y' in data_ismip6.variables: + ismip6_grid_file = args.ismip6_grid_file + print("'x' and 'y' coordinates exist in the file.") + else: + print("'x' and 'y' coordinates don't exist in the file.") + print("Creating them and a copy file of the ismip6 grid file...") + copy_ismip6_file = f"temp_{os.path.basename(args.ismip6_grid_file)}" + shutil.copy2(args.ismip6_grid_file, copy_ismip6_file) + copy_ismip6_file = Dataset(copy_ismip6_file, "r+", format="netCDF4") + nx = data_ismip6.dimensions["x"].size + ny = data_ismip6.dimensions["y"].size + dx = int(args.res_ismip6_grid)*1000 + dy = dx + if (nx % 2) == 0: + var_x = dx*((np.arange(-nx/2, nx/2)) + 0.5) + var_y = dy*((np.arange(-ny/2, ny/2)) + 0.5) + else: + var_x = dx*((np.arange(-(nx-1)/2, (nx+1)/2))) + var_y = dy*((np.arange(-(ny-1)/2, (ny+1)/2))) + + x = copy_ismip6_file.createVariable("x", "d", ("x")) + y = copy_ismip6_file.createVariable("y", "d", ("y")) + + for i in range(nx): + x[i] = var_x[i] + for i in range(ny): + y[i] = var_y[i] + + x.units = 'm' + x.standard_name = 'x' + y.units = 'm' + y.standard_name = 'y' + + copy_ismip6_file.close() + ismip6_grid_file = f"temp_{os.path.basename(args.ismip6_grid_file)}" + temp_ismip6_grid_file = True + + # check the lower left and upper right corners of the ismip6 grid + print("Checking the grid corners...") + data_ismip6 = Dataset(ismip6_grid_file, "r") + x = data_ismip6.variables["x"] + y = data_ismip6.variables["y"] + if not x[0] == -3040000 or not y[0] == -3040000: + raise ValueError(f"The lower left corner values must be at " + f"-3040000m and -3040000m. But the values are at " + f"{x[0]}m and {y[0]}m. Check the value you " + f"provided for '--res' matches with the resolution of " + f"the MALI output files. ") + elif not x[-1] == 3040000 or not y[-1] == 3040000: + raise ValueError(f"The upper right corner values must be at " + f"3040000m and 3040000m. But the values are at " + f"{x[-1]}m and {y[-1]}m. Check the value you " + f"provided for '--res' matches with the resolution of " + f"the MALI output files. ") + else: + print(f"Grid corners are as ismip6-required: " + f"lower right corner values at {x[0]}m and {y[0]}m, and " + f"upper right corner values at {x[-1]}m and {y[-1]}m") + + print("\n---Processing remapping file---") + # Only do remapping steps if we have 2d files to process + if not args.input_file_state is None or not args.input_file_flux is None: + # check the mapping method and existence of the mapping file + # Note: the function 'building_mapping_file' requires the mpas mesh tool + # script 'create_SCRIP_file_from_planar_rectangular_grid.py' + if os.path.exists(args.mapping_file): + print(f"Mapping file exists.") + mapping_file = args.mapping_file + else: + if args.method_remap is None: + method_remap = "conserve" + else: + method_remap = args.method_remap + + mapping_file = f"map_{args.mali_mesh_name}_to_"\ + f"ismip6_{args.res_ismip6_grid}km_{method_remap}.nc" + + print(f"Creating new mapping file." + f"Mapping method used: {method_remap}") + + build_mapping_file(args.input_file_grid, mapping_file, + args.res_ismip6_grid, ismip6_grid_file, + method_remap) + + print("---Processing remapping file complete---\n") + + # define the path to which the output (processed) files will be saved + if args.output_path is None: + output_path = os.getcwd() + else: + output_path = args.output_path + print(f"Using output path: {output_path}") + if not os.path.isdir(output_path): + os.makedirs(output_path) + + if args.input_file_state is None: + print("--- MALI state file is not provided, thus it will not be processed.") + else: + print("\n---Processing state file---") + # state variables processing part + # process (add and rename) state vars as requested by the ISMIP6 protocol + print("Calculating needed state file adjustments.") + tmp_file = "tmp_state.nc" + process_state_vars(args.input_file_state, tmp_file) + + # remap data from the MALI unstructured mesh to the ISMIP6 polarstereo grid + processed_and_remapped_file_state = f'processed_and_remapped_' \ + f'{os.path.basename(args.input_file_state)}' + + print("Remapping state file.") + command = ["ncremap", + "-i", tmp_file, + "-o", processed_and_remapped_file_state, + "-m", mapping_file, + "-P", "mpas"] + check_call(command) + + # write out 2D state output files in the ismip6-required format + print("Writing processed and remapped state fields to ISMIP6 file format.") + generate_output_2d_state_vars(processed_and_remapped_file_state, + ismip6_grid_file, + args.exp, output_path) + + os.remove(tmp_file) + os.remove(processed_and_remapped_file_state) + print("---Processing state file complete---\n") + + # write out 1D output files for both state and flux variables + if args.global_stats_file is None: + print("--- MALI global stats file is not provided, thus it will not be processed.") + else: + print("\n---Processing global stats file---") + generate_output_1d_vars(args.global_stats_file, args.exp, + output_path) + print("---Processing global stats file complete---\n") + + # process the flux variables if flux output file is given + if args.input_file_flux is None: + print("--- MALI flux file is not provided, thus it will not be processed.") + else: + print("\n---Processing flux file---") + + print("Adjusting flux fields that need modification before time averaging.") + tmp_file_translate = "flux_translated.nc" + clean_flux_fields_before_time_averaging(args.input_file_flux, args.input_file_grid, tmp_file_translate) + # take time (yearly) average for the flux variables + tmp_file1 = "flux_time_avg.nc" + do_time_avg_flux_vars(tmp_file_translate, tmp_file1) + + # remap data from the MALI unstructured mesh to the ISMIP6 P-S grid + processed_file_flux = f'processed_' \ + f'{os.path.basename(args.input_file_flux)}' + command = ["ncremap", + "-i", tmp_file1, + "-o", processed_file_flux, + "-m", mapping_file, + "-P", "mpas"] + check_call(command) + + # write out the output files in the ismip6-required format + generate_output_2d_flux_vars(processed_file_flux, + ismip6_grid_file, + args.exp, output_path) + + cleanUp = True + if cleanUp: + os.remove(tmp_file_translate) + os.remove(tmp_file1) + os.remove(processed_file_flux) + if temp_ismip6_grid_file: + os.remove(ismip6_grid_file) + print("---Processing flux file complete---\n") + print("---All processing complete---") + +if __name__ == "__main__": + main() diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables.py new file mode 100644 index 000000000..a528ac642 --- /dev/null +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables.py @@ -0,0 +1,518 @@ +""" +This script has functions that are needed to post-process and write flux +output variables from ISMIP6 simulations. +""" + +from netCDF4 import Dataset +import xarray as xr +import numpy as np +from datetime import date +from subprocess import check_call +import os, sys +import warnings + + +def do_time_avg_flux_vars(input_file, output_file): + """ + input_file: MALI simulation flux file that has the all time levels + output_file: file with time-averaged fluxes + """ + print("Starting time averaging of flux variables") + dataIn = xr.open_dataset(input_file, decode_cf=False) # need decode_cf=False to prevent xarray from reading daysSinceStart as a timedelta type. + if 'units' in dataIn.daysSinceStart.attrs: # make have been removed in a previous step, so check if it exists + del dataIn.daysSinceStart.attrs['units'] # need this line to prevent xarray from reading daysSinceStart as a timedelta type. + + time = dataIn.dims['Time'] + nCells = dataIn.dims['nCells'] + xtimeIn = dataIn['xtime'][:].values + #print(xtimeIn) + xtime = [] + for i in range(time): + xtime.append(xtimeIn[i].tostring().decode('utf-8').strip().strip('\x00')) + #print(xtime) + deltat = dataIn['deltat'][:] + daysSinceStart = dataIn['daysSinceStart'][:] + cellMask = dataIn['cellMask'][:,:] + sfcMassBal = dataIn['sfcMassBalApplied'][:, :] + floatingBasalMassBalApplied = dataIn['floatingBasalMassBalApplied'][:, :] + groundedBasalMassBalApplied = dataIn['groundedBasalMassBalApplied'][:, :] + dHdt = dataIn['dHdt'][:,:] / (3600.0 * 24.0 * 365.0) # convert units to m/s + glFlux = dataIn['fluxAcrossGroundingLineOnCells'][:, :] + calvingFlux = dataIn['calvingFlux'][:, :] + faceMeltAndCalvingFlux = dataIn['faceMeltAndCalvingFlux'][:, :] + + iceMask = (cellMask[:, :] & 2) / 2 # grounded: dynamic ice + + # Figure out some timekeeping stuff - using netCDF4 b/c xarray is a nightmare + fin = Dataset(input_file, 'r') + simulationStartTime = fin.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + fin.close() + simulationStartDate = simulationStartTime.split("_")[0] + if simulationStartDate[5:10] != '01-01': + sys.exit("Error: simulationStartTime for flux file is not on Jan. 1.") + refYear = int(simulationStartDate[0:4]) + startYr = refYear + np.floor(daysSinceStart[0] / 365.0) # using floor here because we might not have output at jan 1, but we'll definitely have at least one time level per year + finalYr = refYear + daysSinceStart[-1] / 365.0 + if (daysSinceStart[-1] / 365.0 != daysSinceStart[-1] // 365): + sys.exit(f"Error: final time of flux output file is not on Jan. 1.: daysSinceStart={daysSinceStart[-1]}, xtime={xtime[-1]}" ) + print(f"simulationStartTime={simulationStartTime}; simulationStartDate={simulationStartDate}; refYear={refYear}") + print(f"start year={startYr}; final year={finalYr}") + + # get an array of years that are not duplicative + decYears = refYear + daysSinceStart/365.0 + #years = np.floor(decYears - 1.0e-10) # this is the "owning" year; Jan 1 belongs to the previous year, so offset decYears by small amount + #years[0] = int(xtime[0].decode("utf-8")[0:4]) + ##years = np.trim_zeros(years) + years = np.arange(startYr, finalYr) # we don't want the final year in the time array as a year to process - it's actually the end point of the previous year + + timeBndsMin = np.ones((len(years),)) * 1.0e36 + timeBndsMax = np.ones((len(years),)) * -1.0e36 + + avgSmb = np.zeros((len(years), nCells)) * np.nan + avgCF = np.zeros((len(years), nCells)) * np.nan + avgCFandFM = np.zeros((len(years), nCells)) * np.nan + avgBmbfl = np.zeros((len(years), nCells)) * np.nan + avgBmbgr = np.zeros((len(years), nCells)) * np.nan + avgDHdt = np.zeros((len(years), nCells)) * np.nan + avgGF = np.zeros((len(years), nCells)) * np.nan + maxIceMask = np.zeros((len(years), nCells), dtype=np.int) * np.nan + + print(" begin looping over years") + for j in range(len(years)): + # we want time bounds to span the full year + timeBndsMin[j] = (years[j] - refYear) * 365.0 + timeBndsMax[j] = (years[j]+1.0 - refYear) * 365.0 + print(f" year index: {j}, year={years[j]}; timeBindsMin={timeBndsMin[j]}, timeBndsMax={timeBndsMax[j]}") + sumYearSmb = 0 + sumYearBmb = 0 + sumYearDHdt = 0 + sumYearCF = 0 + sumYearCFandFM = 0 + sumYearGF = 0 + sumYearBHF = 0 + sumYearTime = 0 + sumYearBmbfl = 0 + sumYearBmbgr = 0 + sumIceMask = 0 + + timeBndMin = 1.0e36 + timeBndMax = -1.0e36 + for i in range(time): + + if decYears[i] > years[j] and decYears[i] <= years[j]+1.0: + sumYearSmb = sumYearSmb + sfcMassBal[i, :] * deltat[i] + sumYearBmbfl = sumYearBmbfl + floatingBasalMassBalApplied[i, :] * deltat[i] + sumYearBmbgr = sumYearBmbgr + groundedBasalMassBalApplied[i, :] * deltat[i] + sumYearDHdt = sumYearDHdt + dHdt[i, :] * deltat[i] + sumYearCF = sumYearCF + calvingFlux[i,:] * deltat[i] + sumYearCFandFM = sumYearCFandFM + faceMeltAndCalvingFlux[i,:] * deltat[i] + sumYearGF = sumYearGF + glFlux[i, :] * deltat[i] + sumYearTime = sumYearTime + deltat[i] + + sumIceMask = sumIceMask + iceMask[i,:] + + print(f" year={years[j]}, decYears={decYears[i]}, daysSinceStart={daysSinceStart[j]}, xtime={xtime[i]}") + + + avgSmb[j,:] = sumYearSmb / sumYearTime + avgBmbfl[j,:] = sumYearBmbfl / sumYearTime + avgBmbgr[j,:] = sumYearBmbgr / sumYearTime + avgDHdt[j,:] = sumYearDHdt / sumYearTime + avgCF[j,:] = sumYearCF / sumYearTime + avgCFandFM[j,:] = sumYearCFandFM / sumYearTime + avgGF[j,:] = sumYearGF / sumYearTime + maxIceMask[j,:] = (sumIceMask>0) # Get mask for anywhere that had ice during this year + + + print(" write time averaged values") + + print(f"avg shape={avgSmb.shape}, time shape={timeBndsMin.shape}") + out_data_vars = { + 'sfcMassBalApplied': (['Time', 'nCells'], avgSmb), + 'libmassbffl': (['Time', 'nCells'], avgBmbfl), + 'libmassbfgr': (['Time', 'nCells'], avgBmbgr), + 'dHdt': (['Time', 'nCells'], avgDHdt), + 'fluxAcrossGroundingLineOnCells': (['Time', 'nCells'], avgGF), + 'calvingFlux': (['Time', 'nCells'], avgCF), + 'faceMeltAndCalvingFlux': (['Time', 'nCells'], avgCFandFM), + 'iceMask': (['Time', 'nCells'], maxIceMask), + 'timeBndsMin': (['Time'], timeBndsMin), + 'timeBndsMax': (['Time'], timeBndsMax), + 'simulationStartTime': dataIn['simulationStartTime'] + } + out_coords = { + 'Time': (['Time'], (timeBndsMin+timeBndsMax)/2.0) + } + + + dataOut = xr.Dataset(data_vars=out_data_vars, coords=out_coords) + dataOut.to_netcdf(output_file, mode='w') + dataIn.close() + + +def clean_flux_fields_before_time_averaging(file_input, file_mesh, + file_output): + """ + Convert the MALI output field calvingThickness to the ISMIP6 variable + licalvf and apply bounds checking on BMB, where some crazy values occasionally occur. + """ + + debug_face_melt_flux = False + data = xr.open_dataset(file_input, decode_cf=False) # need decode_cf=False to prevent xarray from reading daysSinceStart as a timedelta type. + if 'units' in data.daysSinceStart.attrs: + del data.daysSinceStart.attrs['units'] # need this line to prevent xarray from reading daysSinceStart as a timedelta type. + time = data.dims['Time'] + nCells = data.dims['nCells'] + nEdgesOnCell = data['nEdgesOnCell'][:].values + edgesOnCell = data['edgesOnCell'][:].values + cellsOnCell = data['cellsOnCell'][:].values + dvEdge = data['dvEdge'][:].values + areaCell = data['areaCell'][:].values + xCell = data['xCell'][:].values + yCell = data['yCell'][:].values + deltat = data['deltat'][:].values + thickness = data['thickness'][:].values + surfaceSpeed = data['surfaceSpeed'][:].values + if 'bedTopography' in data: + bedTopography = data['bedTopography'][:].values + print('bedTopography field found; using bedTopography at all time levels.') + else: + data_mesh = xr.open_dataset(file_mesh) + bedTopography = data_mesh['bedTopography'][:].values + print('No bedTopography field found; using bedTopography from mesh file.') + if 'calvingThicknessFromThreshold' in data: + calvingThicknessFromThreshold = data['calvingThicknessFromThreshold'][:, :].values + else: + print('WARNING: No calvingThicknessFromThreshold field found; creating a field populated with zeros.') + calvingThicknessFromThreshold = thickness.copy() * 0.0 + + rho_i = 910.0 + + print("===starting cleaning floatingBasalMassBalApplied===") + # We've encountered a few enormous BMB values. Until we solve where that is coming from, + # set them to something reasonable. + floatingBasalMassBalApplied = data['floatingBasalMassBalApplied'][:, :].values + for t in range(1, time): + if t%20 == 0: + print(f" Time: {t+1} / {time}") + # Set large negative BMB values (1 m/s) to the equivalent of the thickness from the previous time step + # (Commented line here picked up too many places) + #ind = np.nonzero((-floatingBasalMassBalApplied[t,:]/rho_i*deltat[t] > thickness[t-1,:]) * (thickness[t-1,:]>0.0))[0] + ind = np.nonzero(floatingBasalMassBalApplied[t,:]/rho_i < -1.0)[0] + if len(ind) > 0: + print(f"Fixing {len(ind)} cells with large negative floating BMB values.", ind) + floatingBasalMassBalApplied[t, ind] = -thickness[t-1, ind] / deltat[t] * rho_i + + ind = np.nonzero(floatingBasalMassBalApplied[t,:]/rho_i > 1.0)[0] + if len(ind) > 0: + print(f"Fixing {len(ind)} cells with large positive floating BMB values.", ind) + ind2 = np.nonzero(floatingBasalMassBalApplied[t,:]/rho_i <= 1.0)[0] + maxGoodBMB = floatingBasalMassBalApplied[t, ind2].max() + floatingBasalMassBalApplied[t, ind] = maxGoodBMB + + print("===done cleaning floatingBasalMassBalApplied===") + + assert time == len(deltat) + + calvingVelocity = data['calvingVelocity'][:, :].values + + # create and initialize a new data array for calvingFluxArray + calvingFluxArray = data['calvingVelocity'].copy() * 0.0 + thresholdFlux = data['calvingVelocity'].copy() * 0.0 + calvingThickness = data['calvingThickness'][:, :].values + print("===starting facemelt flux processing===") + + # create and initialize a new data array for faceMeltFluxArray + # (copied from calving code below) + # Some runs won't have this output field, so assume if field is not present + # that facemelting was not enabled + faceMeltFluxArray = data['calvingVelocity'].copy() * 0.0 + if 'faceMeltSpeed' in data: + faceMeltSpeed = data['faceMeltSpeed'][:, :].values + # faceMeltSpeed is defined below the water line, but face-melting is + # applied to the full ice thickness, so the effective speed is + # averaged over the full thickness from the previous time step. + # Note that this calculation assumes that bedTopography is constant in time, + # that config_sea_level = 0, and that faceMeltSpeed is only valid for + # grounded cells, i.e., that bedTopography and lowerSurface are equivalent + # (which is currently the case). + + faceMeltingThickness = data['faceMeltingThickness'][:, :].values + faceMeltSpeedVertAvg = faceMeltingThickness.copy() * 0.0 + # Fields for validation and debugging + if debug_face_melt_flux: + deltat_array = np.tile(deltat, (np.shape(faceMeltSpeed)[1],1)).transpose() + # Cleaned field for debugging and validation + faceMeltingThicknessCleaned = faceMeltingThickness.copy() + for t in range(time): + if t%20 == 0: + print(f" Time: {t+1} / {time}") + + if 'bedTopography' in data: + bed = bedTopography[t,:] # have value per time level + else: + bed = bedTopography[0,:] # just have a single value + + prev_t = max(t-1, 0) # ensure that index_cf never uses thickness from last (-1) time step + index_cf = np.where((faceMeltingThickness[t, :] > 0.0) * (bed[:] < 0.0) * + (faceMeltingThickness[t, :] != thickness[prev_t, :]) * + (thickness[prev_t, :] > 0.))[0] + for i in index_cf: + # faceMeltSpeed is calculated for ice below water line, but needs to be aplied + # to full ice thickness, so we need a vertically averaged speed. Also ensure that + # the vertically averaged speed is never > faceMeltSpeed due to small ice thickness. + # This may be slightly inaccurate on the very first time step. + faceMeltSpeedVertAvg[t,i] = faceMeltSpeed[t, i] * np.abs(bed[i] / thickness[prev_t, i]) + faceMeltSpeedVertAvg[t,i] = min(faceMeltSpeedVertAvg[t,i], faceMeltSpeed[t, i]) + # Use this cell if it has nonzero faceMeltingThickness because faceMeltSpeed + # is defined everywhere, but only applied on grounded ice + if faceMeltingThickness[t,i] > 0.0: + faceMeltFluxArray[t,i] = faceMeltSpeedVertAvg[t,i] * rho_i # convert to proper units + # Push mass removed from stranded non-dynamic cells into calving + index_stranded_cell_cleanup = np.where(faceMeltingThickness[t, :] == thickness[prev_t, :])[0] + calvingThicknessFromThreshold[t, index_stranded_cell_cleanup] += faceMeltingThickness[t, index_stranded_cell_cleanup] + if debug_face_melt_flux: + faceMeltingThicknessCleaned[t, index_stranded_cell_cleanup] -= faceMeltingThickness[t, index_stranded_cell_cleanup] + # This is just for debugging and validation + print("===done facemelt flux processing!===") + + print("===starting the calving flux processing===") + + for t in range(time): + if t%20 == 0: + print(f" Time: {t+1} / {time}") + + if 'bedTopography' in data: + bed = bedTopography[t,:] # have value per time level + else: + bed = bedTopography[0,:] # just have a single value + + index_cf = np.where((calvingVelocity[t, :] > 0.0) * (bed[:] < 0.0))[0] + for i in index_cf: + ne = nEdgesOnCell[i] + for j in range(ne): + neighborCellId = cellsOnCell[i, j] - 1 + # Use this cell if it has a neighbor with zero calvingVelocity that is below sea level + if calvingVelocity[t,neighborCellId] == 0.0 and bed[neighborCellId] < 0.0: + calvingFluxArray[t,i] = calvingVelocity[t,i] * rho_i # convert to proper units + continue # no need to keep searching the neighbors of this cell + + + # we may need to add on threshold calving too + if 'calvingThicknessFromThreshold' in data: + index_cf = np.where(calvingThicknessFromThreshold[t, :] > 0.0)[0] + else: + index_cf = [] + if len(index_cf) > 0: + thresholdBoundary = np.zeros((nCells,), 'i') + thresholdBoundaryAssignedVolume = np.zeros((nCells,)) + thresholdBoundarySummedThickness = np.zeros((nCells,)) + thresholdBoundaryContributors = np.zeros((nCells,)) + thresholdBoundaryLength = np.zeros((nCells,)) + thresholdSpeed = np.zeros((nCells,)) + # First make list of boundary cells calved + for i in index_cf: + ne = nEdgesOnCell[i] + for j in range(ne): + neighborCellId = cellsOnCell[i, j] - 1 + if thickness[t,neighborCellId] > 0.0 and bed[neighborCellId] < 0.0 and calvingThicknessFromThreshold[t,neighborCellId] == 0.0: + thresholdBoundary[i] = 1 + thresholdBoundaryLength[i] += dvEdge[edgesOnCell[i,j]-1] + bdyIndices = np.where(thresholdBoundary == 1)[0] + print(f"Found {len(index_cf)} cells with threshold calving at time {t}; {len(bdyIndices)} are boundary cells.") + if len(bdyIndices) == 0: + print(f"0 boundary cells were found; skipping to next time step") + continue + # Now loop over all threshold cells and assign their volume to the nearest boundary cell + for i in index_cf: + if thresholdBoundary[i] == 1: + ownerIdx = i # often the cell is its own owner, so check before doing the more expensive search + else: + ownerIdx = bdyIndices[np.argmin((xCell[i]-xCell[bdyIndices])**2 + (yCell[i]-yCell[bdyIndices])**2)] + thresholdBoundaryAssignedVolume[ownerIdx] += calvingThicknessFromThreshold[t,i] * areaCell[i] + thresholdBoundarySummedThickness[ownerIdx] += calvingThicknessFromThreshold[t,i] + thresholdBoundaryContributors[ownerIdx] += 1 + #print(thresholdBoundaryAssignedVolume.sum(), (calvingThicknessFromThreshold[t,:]*areaCell[:]).sum()) + diff = np.absolute(thresholdBoundaryAssignedVolume.sum() - (calvingThicknessFromThreshold[t,:]*areaCell[:]).sum()) + if diff < 1.0: + warnings.warn(f"Difference between assigned `thresholdBoundaryAssignedVolume` value and " + f"`calvingThicknessFromThreshold` threshold value is less than 1: {diff} < 1.0") + #for i in bdyIndices: + #print(f"length={thresholdBoundaryLength[i]}, vol={thresholdBoundaryAssignedVolume[i]}, sumthk={thresholdBoundarySummedThickness[i]}, num={thresholdBoundaryContributors[i]}, meanthk={thresholdBoundarySummedThickness[i]/thresholdBoundaryContributors[i]}") + # Finally calculate licalvf for each boundary cell and add to whatever was already there + thresholdSpeed[bdyIndices] = thresholdBoundaryAssignedVolume[bdyIndices] / \ + (thresholdBoundarySummedThickness[bdyIndices] / thresholdBoundaryContributors[bdyIndices] * \ + thresholdBoundaryLength[bdyIndices]) / \ + deltat[t] # units of m/s + # Our estimated threshold speed is really a retreat speed. So to get calving speed, add on the advective speed + thresholdFlux[t, bdyIndices] += (thresholdSpeed[bdyIndices] + surfaceSpeed[t,bdyIndices]) * rho_i + calvingFluxArray[t,bdyIndices] += thresholdFlux[t,bdyIndices] + + data['calvingFlux'] = calvingFluxArray # Note: thresholdFlux was already added in above + data['thresholdFlux'] = thresholdFlux # this is just written for diagnostic purposes. It's not actually sent to ISMIP6. + data['faceMeltAndCalvingFlux'] = faceMeltFluxArray + calvingFluxArray # ismip6 only wants the combined fields for face-melt + print("===done calving flux processing!===") + if debug_face_melt_flux: + print('debug_face_melt_flux is True, so I assume you want a breakpoint' + + ' to check fluxes. Just type continue when you want to proceed.') + breakpoint() + data.to_netcdf(file_output) # copy of the input file with new vars added + data.close() + +def write_netcdf_2d_flux_vars(mali_var_name, ismip6_var_name, var_std_name, + var_units, var_varname, remapped_mali_flux_file, + ismip6_grid_file, exp, output_path): + + """ + mali_var_name: variable name on MALI side + ismip6_var_name: variable name required by ISMIP6 + var_std_name: standard variable name + var_units: variable units + var_varname: variable variable name + remapped_mali_flux_file: mali flux file remapped on the ISMIP6 grid + ismip6_grid_file: original ISMIP6 file + exp: experiment name + output_path: output path to which the output files will be saved + """ + + data_ismip6 = Dataset(ismip6_grid_file, 'r') + var_x = data_ismip6.variables['x'][:] + var_y = data_ismip6.variables['y'][:] + + data = Dataset(remapped_mali_flux_file, 'r') + data.set_auto_mask(False) + iceMask = data.variables['iceMask'][:, :, :] + simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + simulationStartDate = simulationStartTime.split("_")[0] + timeBndsMin = data.variables['timeBndsMin'][:] + timeBndsMax = data.variables['timeBndsMax'][:] + if not mali_var_name in data.variables: + print(f"WARNING: {mali_var_name} not present. Skipping.") + data.close() + return + var_mali = data.variables[mali_var_name][:,:,:] + var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN + timeSteps, latN, lonN = np.shape(var_mali) + + dataOut = Dataset(f'{output_path}/{ismip6_var_name}_AIS_DOE_MALI_{exp}.nc', + 'w', format='NETCDF4_CLASSIC') + dataOut.createDimension('time', timeSteps) + dataOut.createDimension('bnds', 2) + timebndsValues = dataOut.createVariable('time_bnds', 'd', ('time', 'bnds')) + dataOut.createDimension('x', lonN) + dataOut.createDimension('y', latN) + dataValues = dataOut.createVariable(ismip6_var_name, 'd', + ('time', 'y', 'x'), fill_value=np.NAN) + xValues = dataOut.createVariable('x', 'd', ('x')) + yValues = dataOut.createVariable('y', 'd', ('y')) + timeValues = dataOut.createVariable('time', 'd', ('time')) + + AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' + DATE_STR = date.today().strftime("%d-%b-%Y") + + for i in range(timeSteps): + mask = iceMask[i, :, :] + tmp = var_mali[i, :, :] + tmp[mask == 0] = np.NAN + dataValues[i, :, :] = tmp + timeValues[i] = (timeBndsMin[i] + timeBndsMax[i]) / 2.0 + timebndsValues[i, 0] = timeBndsMin[i] + timebndsValues[i, 1] = timeBndsMax[i] + + for i in range(latN): + xValues[i] = var_x[i] + + for i in range(lonN): + yValues[i] = var_y[i] + + dataValues.standard_name = var_std_name + dataValues.units = var_units + timeValues.bounds = 'time_bnds' + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + timebndsValues.units = f'days since {simulationStartDate}' + timebndsValues.calendar = 'noleap' + xValues.units = 'm' + xValues.standard_name = 'x' + xValues.long_name = 'x' + yValues.units = 'm' + yValues.standard_name = 'y' + yValues.long_name = 'y' + dataOut.AUTHORS = AUTHOR_STR + dataOut.MODEL = 'MALI (MPAS-Albany Land Ice model)' + dataOut.GROUP = 'Los Alamos National Laboratory, Department of Energy' + dataOut.VARIABLE = var_varname + dataOut.DATE = DATE_STR + dataOut.close() + data.close() + + +def generate_output_2d_flux_vars(file_remapped_mali_flux, + ismip6_grid_file, exp, output_path): + """ + file_remapped_mali_flux: flux output file on mali mesh remapped + onto the ismip6 grid + ismip6 grid + ismip6_grid_file: ismip6 original file + exp: ISMIP6 experiment name + output_path: path to which the final output files are saved + """ + + print("Writing 2d flux variables") + # ----------- acabf ------------------ + write_netcdf_2d_flux_vars('sfcMassBalApplied', 'acabf', + 'land_ice_surface_specific_mass_balance_flux', + 'kg m-2 s-1', 'Surface mass balance flux', + file_remapped_mali_flux, + ismip6_grid_file, exp, output_path) + + # ----------- libmassbffl ------------------ + write_netcdf_2d_flux_vars('libmassbffl', 'libmassbffl', + 'land_ice_basal_specific_mass_balance_flux', + 'kg m-2 s-1', + 'Basal mass balance flux beneath floating ice', + file_remapped_mali_flux, + ismip6_grid_file, exp, output_path) + + # ----------- libmassbfgr ------------------ + write_netcdf_2d_flux_vars('libmassbfgr', 'libmassbfgr', + 'land_ice_basal_specific_mass_balance_flux', + 'kg m-2 s-1', + 'Basal mass balance flux beneath grounded ice', + file_remapped_mali_flux, + ismip6_grid_file, exp, output_path) + + # ----------- dlithkdt ------------------ + write_netcdf_2d_flux_vars('dHdt', 'dlithkdt', + 'tendency_of_land_ice_thickness', + 'm s-1', + 'Ice thickness imbalance', + file_remapped_mali_flux, + ismip6_grid_file, exp, output_path) + + # ----------- licalvf ------------------ + write_netcdf_2d_flux_vars('calvingFlux', 'licalvf', + 'land_ice_specific_mass_flux_due_to_calving', + 'kg m-2 s-1', + 'Calving flux', + file_remapped_mali_flux, + ismip6_grid_file, exp, output_path) + + # ----------- lifmassbf ------------------ + # Note: facemelting and calving flux are combined above + write_netcdf_2d_flux_vars('faceMeltAndCalvingFlux', 'lifmassbf', + 'land_ice_specific_mass_flux_due_to_calving_and_ice_front_melting', + 'kg m-2 s-1', + 'Ice front melt and calving flux', + file_remapped_mali_flux, + ismip6_grid_file, exp, output_path) + + # ----------- ligroundf ------------------ + write_netcdf_2d_flux_vars('fluxAcrossGroundingLineOnCells', 'ligroundf', + 'land_ice_specific_mass_flux_at_grounding_line', + 'kg m-2 s-1', + 'Grounding line flux', + file_remapped_mali_flux, + ismip6_grid_file, exp, output_path) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables.py new file mode 100755 index 000000000..6e8388d3f --- /dev/null +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables.py @@ -0,0 +1,675 @@ +""" +This script has functions that are needed to post-process and write state +output variables from ISMIP6 simulations. +The input files (i.e., MALI output files) need to have been +concatenated to have yearly data, which can be done using 'ncrcat' command +before using this script. +""" + +from netCDF4 import Dataset +import xarray as xr +import numpy as np +from datetime import date +import shutil +import os, sys + + +def process_state_vars(inputfile_state, tmp_file): + """ + inputfile_state: output file copy from MALI simulations + tmp_file: temporary file name + inputfile_temperature: output temperature file from MALI simulations + """ + + inputfile_state_vars = xr.open_dataset(inputfile_state, engine="netcdf4", decode_cf=False) + del inputfile_state_vars.daysSinceStart.attrs['units'] # need this line to prevent xarray from reading daysSinceStart as a timedelta type and corrupting values after about 250 years + + # get the mesh description data + nCells = inputfile_state_vars.dims['nCells'] + nTime = inputfile_state_vars.dims['Time'] + nLayer = inputfile_state_vars.dims['nVertLevels'] + nInterface = nLayer + 1 # inputfile_state_vars.dims['nVertInterfaces'] + cellMask = inputfile_state_vars['cellMask'][:, :] + basalTemperature = inputfile_state_vars['basalTemperature'][:, :] + betaSolve = inputfile_state_vars['betaSolve'][:, :] + + inputfile_state_vars['litempbotfl'] = basalTemperature * (cellMask[:, :] & 4) / 4 + inputfile_state_vars['litempbotgr'] = basalTemperature * (1 - (cellMask[:, :] & 4) / 4) + + uxsurf = inputfile_state_vars['uReconstructX'][:, :, 0] + uysurf = inputfile_state_vars['uReconstructY'][:, :, 0] + uxbase = inputfile_state_vars['uReconstructX'][:, :, nInterface - 1] + uybase = inputfile_state_vars['uReconstructY'][:, :, nInterface - 1] + inputfile_state_vars['uReconstructX_sfc'] = uxsurf + inputfile_state_vars['uReconstructY_sfc'] = uysurf + inputfile_state_vars['uReconstructX_base'] = uxbase + inputfile_state_vars['uReconstructY_base'] = uybase + + inputfile_state_vars['upperSurface'] = np.maximum(0.0, inputfile_state_vars['upperSurface']) + + inputfile_state_vars['sftflf'] = (cellMask[:, :] & 4) / 4 * (cellMask[:, :] & 2) / 2 # floating and dynamic + inputfile_state_vars['sftgrf'] = ((cellMask[:, :] * 0 + 1) - (cellMask[:, :] & 4) / 4) * (cellMask[:, :] & 2) / 2 # grounded: not-floating & dynamic + inputfile_state_vars['sftgif'] = (cellMask[:, :] & 2) / 2 # grounded: dynamic ice + inputfile_state_vars['strbasemag'] = betaSolve[:, :] * ((uxbase[:, :]) ** 2 + (uybase[:, :]) ** 2) **0.5 \ + * (3600.0 * 24.0 * 365.0) \ + * (cellMask[:, :] * 0 + 1 - (cellMask[:, :] & 4) / 4) * (cellMask[:, :] & 2) / 2 + + inputfile_state_vars.to_netcdf(tmp_file) + inputfile_state_vars.close() + + +def write_netcdf_2d_state_vars(mali_var_name, ismip6_var_name, var_std_name, + var_units, var_varname, remapped_mali_outputfile, + ismip6_grid_file, exp, output_path): + """ + mali_var_name: variable name on MALI side + ismip6_var_name: variable name required by ISMIP6 + var_std_name: standard variable name + var_units: variable units + var_varname: variable variable name + remapped_mali_outputfile: mali state file remapped on the ISMIP6 grid + ismip6_grid_file: original ISMIP6 file + exp: experiment name + output_path: output path to which the output files will be saved + """ + + data_ismip6 = Dataset(ismip6_grid_file, 'r') + var_x = data_ismip6.variables['x'][:] + var_y = data_ismip6.variables['y'][:] + + data = Dataset(remapped_mali_outputfile, 'r') + data.set_auto_mask(False) + simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + simulationStartDate = simulationStartTime.split("_")[0] + daysSinceStart = data.variables['daysSinceStart'][:] + var_sftgif = data.variables['sftgif'][:, :, :] + var_sftgrf = data.variables['sftgrf'][:, :, :] + var_sftflf = data.variables['sftflf'][:, :, :] + var_mali = data.variables[mali_var_name][:,:,:] + var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN + timeSteps, latN, lonN = np.shape(var_mali) + + dataOut = Dataset(f'{output_path}/{ismip6_var_name}_AIS_DOE_MALI_{exp}.nc', + 'w', format='NETCDF4_CLASSIC') + dataOut.createDimension('time', timeSteps) + dataOut.createDimension('x', lonN) + dataOut.createDimension('y', latN) + dataValues = dataOut.createVariable(ismip6_var_name, 'd', + ('time', 'y', 'x'), fill_value=np.NAN) + xValues = dataOut.createVariable('x', 'd', ('x')) + yValues = dataOut.createVariable('y', 'd', ('y')) + timeValues = dataOut.createVariable('time', 'd', ('time')) + timeValues[:] = daysSinceStart + AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' + DATE_STR = date.today().strftime("%d-%b-%Y") + + for i in range(timeSteps): + if ismip6_var_name == 'sftgif': + dataValues[i, :, :] = var_mali[i, :, :] + else: + if ismip6_var_name == 'litempbotgr': + mask = var_sftgrf[i, :, :] + elif ismip6_var_name == 'litempbotfl': + mask = var_sftflf[i, :, :] + elif ismip6_var_name == 'topg': + mask = np.ones(var_mali.shape[1:]) # don't mask topg + else: + mask = var_sftgif[i, :, :] + tmp = var_mali[i, :, :] + tmp[mask == 0] = np.NAN + dataValues[i, :, :] = tmp + + for i in range(latN): + xValues[i] = var_x[i] + + for i in range(lonN): + yValues[i] = var_y[i] + + dataValues.standard_name = var_std_name + dataValues.units = var_units + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + xValues.units = 'm' + xValues.standard_name = 'x' + xValues.long_name = 'x' + yValues.units = 'm' + yValues.standard_name = 'y' + yValues.long_name = 'y' + dataOut.AUTHORS = AUTHOR_STR + dataOut.MODEL = 'MALI (MPAS-Albany Land Ice model)' + dataOut.GROUP = 'Los Alamos National Laboratory, Department of Energy' + dataOut.VARIABLE = var_varname + dataOut.DATE = DATE_STR + dataOut.close() + + +def generate_output_2d_state_vars(file_remapped_mali_state, + ismip6_grid_file, exp, output_path): + """ + file_remapped_mali_state: output files on mali mesh remapped + on the ismip6 grid + ismip6_grid_file: ismip6 original file + exp: ISMIP6 experiment name + output_path: path to which the final output files are saved + """ + + + # ----------- lithk ------------------ + write_netcdf_2d_state_vars('thickness','lithk', 'land_ice_thickness', + 'm', 'Ice thickness', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- orog ------------------ + write_netcdf_2d_state_vars('upperSurface','orog', 'surface_altitude', 'm', + 'Surface elevation', + file_remapped_mali_state, + ismip6_grid_file,exp, output_path) + + # ----------- base ------------------ + write_netcdf_2d_state_vars('lowerSurface','base', 'base_altitude', 'm', + 'Base elevation', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- topg ------------------ + write_netcdf_2d_state_vars('bedTopography','topg', 'bedrock_altitude', 'm', + 'Bedrock elevation', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- hfgeoubed------------------ + # Note: even though this is a flux variable, we are taking a snapshot of it + # as it does not change with time #Uncomment the function all once basalHeatFlux is outputted in the output stream + # write_netcdf_2d_state_vars('basalHeatFlux', 'hfgeoubed', + # 'upward_geothermal_heat_flux_in_land_ice', + # 'W m-2', 'Geothermal heat flux', + # file_remapped_mali_state, ismip6_grid_file, + # exp, output_path) + + # ----------- xvelsurf ------------------ + write_netcdf_2d_state_vars('uReconstructX_sfc', 'xvelsurf', + 'land_ice_surface_x_velocity', + 'm s-1', 'Surface velocity in x', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # -----------yxvelsurf ------------------ + write_netcdf_2d_state_vars('uReconstructY_sfc', 'yvelsurf', + 'land_ice_surface_y_velocity', + 'm s-1', 'Surface velocity in x', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- xvelbase ------------------ + write_netcdf_2d_state_vars('uReconstructX_base', 'xvelbase', + 'land_ice_basal_x_velocity', + 'm s-1', 'Basal velocity in x', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- yvelbase ------------------ + write_netcdf_2d_state_vars('uReconstructY_base', 'yvelbase', + 'land_ice_basal_y_velocity', + 'm s-1', 'Basal velocity in y', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- zvelsurf & zvelbase ------------------ + # ISMIP6 requires these variables, but MALI does not output them. + # So, we are not processing/writing these variables out. + + # ----------- xvelmean ------------------ + write_netcdf_2d_state_vars('xvelmean', 'xvelmean', + 'land_ice_vertical_mean_x_velocity', + 'm s-1', 'Mean velocity in x', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- yvelmean ------------------ + write_netcdf_2d_state_vars('yvelmean', 'yvelmean', + 'land_ice_vertical_mean_y_velocity', + 'm s-1', 'Mean velocity in y', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- litemptop ------------------ + write_netcdf_2d_state_vars('surfaceTemperature', 'litemptop', + 'temperature_at_top_of_ice_sheet_model', 'K', + 'Surface temperature', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- litempbotgr ------------------ + write_netcdf_2d_state_vars('litempbotgr', 'litempbotgr', + 'temperature_at_base_of_ice_sheet_model', 'K', + 'Basal temperature beneath grounded ice sheet', + file_remapped_mali_state, + ismip6_grid_file,exp, output_path) + + # ----------- litempbotfl ------------------ + write_netcdf_2d_state_vars('litempbotfl', 'litempbotfl', + 'temperature_at_base_of_ice_sheet_model', 'K', + 'Basal temperature beneath floating ice shelf', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- strbasemag ------------------ + write_netcdf_2d_state_vars('strbasemag', 'strbasemag', + 'land_ice_basal_drag ', 'Pa', + 'Basal drag', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- sftgif ------------------ + write_netcdf_2d_state_vars('sftgif','sftgif', + 'land_ice_area_fraction', '1', + 'Land ice area fraction', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- sftgrf ------------------ + write_netcdf_2d_state_vars('sftgrf', 'sftgrf', + 'grounded_ice_sheet_area_fraction', '1', + 'Grounded ice sheet area fraction', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + # ----------- sftflf ------------------ + write_netcdf_2d_state_vars('sftflf','sftflf', + 'floating_ice_shelf_area_fraction', '1', + 'Floating ice shelf area fraction', + file_remapped_mali_state, + ismip6_grid_file, exp, output_path) + + +def generate_output_1d_vars(global_stats_file, exp, output_path=None): + """ + This code processes both state and flux 1D variables + global_stats_file: MALI globalStats.nc output file + exp: ISMIP6 experiment number + output_path: + """ + + if not os.path.exists(output_path): + output_path = os.getcwd() + + AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' + DATE_STR = date.today().strftime("%d-%b-%Y") + + data = Dataset(global_stats_file, 'r') + nt_in = len(data.dimensions['Time']) + xtime = data.variables['xtime'][:, :] + daysSinceStart = data.variables['daysSinceStart'][:] + dt = data.variables['deltat'][:] + simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + simulationStartDate = simulationStartTime.split("_")[0] + if simulationStartDate[5:10] != '01-01': + sys.exit("Error: simulationStartTime for globalStats file is not on Jan. 1.") + refYear = int(simulationStartDate[0:4]) + decYears = refYear + daysSinceStart/365.0 + endYr = decYears[-1] + if endYr != np.round(endYr): + sys.exit("Error: end year not an even year in globalStats file.") + + # Determine processed time levels for state and flux fields + # The historical state fields should include the initial time (Jan. 1). + # Projection state fields should not include the initial time (Jan. 1) + # of the projection because it's a restart from the historical. + # Flux fields should never use the Jan. 1 time level at the start of the + # year as part of the averaging. + # For year conventions here, for state fields, the year is the snapshot at + # the start of the year, e.g., state year 2000 means the snapshot at Jan. 1, 2000. + # For flux fields, the years is the calendar year being averaged over, + # e.g., flux year 2000 is the average between Jan. 1, 2000, and Jan. 1, 2001. + # Note this year convention differs from the first column in table in A2.3.2 at + # https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP6-Projections2300-Antarctica#A2.3.3_Table_A1:_Variable_request_for_ISMIP6 + # but that year indexing convention ultimately doesn't matter because the + # time coordinates in these files uses units of days since a reference date, + # and it does not use a year indexing convention at all. + if decYears[0] == np.round(decYears[0]): + # The initial time level will only be on an even year (Jan. 1) + # for the hist run. In that case, we want to include that initial + # even year in the state processing. We also want the state snapshot + # at the final (even) year in the output. + # The flux processing should start with the first year, which covers a + # full 12 months. We exclude the final year, which is just a Jan. 1 posting. + years_state = np.arange(decYears[0], endYr + 1) + years_flux = np.arange(decYears[0], endYr) + else: + # For projection runs, the first state snapshot we want is the first Jan. 1, + # which we be the first even year after the initial time in the file. + # For flux files, the first full year we want to process is the year of the + # first time level in the file. As with hist, we exclude the final year, + # which is just a Jan. 1 posting. + years_state = np.arange(np.ceil(decYears[0]), endYr + 1) + years_flux = np.arange(np.floor(decYears[0]), endYr) + nt_state = len(years_state) + nt_flux = len(years_flux) + print(f'For state processing, using start year={years_state[0]} and end year={years_state[-1]}.') + print(f'For flux processing, using start year={years_flux[0]} and end year={years_flux[-1]}.') + + # read in state variables + vol = data.variables['totalIceVolume'][:] + vaf = data.variables['volumeAboveFloatation'][:] + gia = data.variables['groundedIceArea'][:] + fia = data.variables['floatingIceArea'][:] + + # read in flux variables over which yearly average will be taken + smb = data.variables['totalSfcMassBal'][:] + bmb = data.variables['totalBasalMassBal'][:] + # clean out some garbage values we can't account for + ind = np.nonzero(bmb>1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalBasalMassBal>1.0e18") + bmb[ind] = np.nan + ind = np.nonzero(bmb<-1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalBasalMassBal<-1.0e18") + bmb[ind] = np.nan + bmbFlt = data.variables['totalFloatingBasalMassBal'][:] + # clean out some garbage values we can't account for + ind = np.nonzero(bmbFlt>1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal>1.0e18") + bmbFlt[ind] = np.nan + ind = np.nonzero(bmbFlt<-1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal<-1.0e18") + bmbFlt[ind] = np.nan + cfx = data.variables['totalCalvingFlux'][:] + fmfx = data.variables['totalFaceMeltingFlux'][:] + gfx = data.variables['groundingLineFlux'][:] + + # initialize 1D variables that will store data value on the + # January 1st of each year + vol_snapshot = np.zeros(nt_state) * np.nan + vaf_snapshot = np.zeros(nt_state) * np.nan + gia_snapshot = np.zeros(nt_state) * np.nan + fia_snapshot = np.zeros(nt_state) * np.nan + days_snapshot = np.zeros(nt_state) * np.nan + smb_avg = np.zeros(nt_flux) * np.nan + bmb_avg = np.zeros(nt_flux) * np.nan + bmbFlt_avg = np.zeros(nt_flux) * np.nan + cfx_avg = np.zeros(nt_flux) * np.nan + cfmfx_avg = np.zeros(nt_flux) * np.nan + gfx_avg = np.zeros(nt_flux) * np.nan + days_min = np.zeros(nt_flux) * np.nan + days_max = np.zeros(nt_flux) * np.nan + + # this is for the state variables + for i in range(nt_state): + ind_snap = np.where(decYears==years_state[i])[0] + + vol_snapshot[i] = vol[ind_snap] + vaf_snapshot[i] = vaf[ind_snap] + gia_snapshot[i] = gia[ind_snap] + fia_snapshot[i] = fia[ind_snap] + days_snapshot[i] = daysSinceStart[ind_snap] + + if decYears[ind_snap] == endYr: + break + + # this is for the flux variables + for i in range(nt_flux): + ind_avg = np.where(np.logical_and(decYears > years_flux[i], + decYears <= (years_flux[i] + 1.0)))[0] + smbi = smb[ind_avg] + bmbi = bmb[ind_avg] + bmbFlti = bmbFlt[ind_avg] + cfxi = cfx[ind_avg] + cfmfxi = cfx[ind_avg] + fmfx[ind_avg] + gfxi = gfx[ind_avg] + dti = dt[ind_avg] + + # take the average of the flux variables + smb_avg[i] = np.nansum(smbi * dti) / np.nansum(dti) + bmb_avg[i] = np.nansum(bmbi * dti) / np.nansum(dti) + bmbFlt_avg[i] = np.nansum(bmbFlti * dti) / np.nansum(dti) + cfx_avg[i] = np.nansum(cfxi * dti) / np.nansum(dti) + cfmfx_avg[i] = np.nansum(cfmfxi * dti) / np.nansum(dti) + gfx_avg[i] = np.nansum(gfxi * dti) / np.nansum(dti) + days_min[i] = (years_flux[i] - refYear) * 365.0 + days_max[i] = (years_flux[i] + 1.0 - refYear) * 365.0 + + if decYears[ind_avg][-1] == endYr: + break + + # -------------- lim ------------------ + data_scalar = Dataset(f'{output_path}/lim_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + limValues = data_scalar.createVariable('lim', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + limValues[i] = vol_snapshot[i] * 910 + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + limValues.standard_name = 'land_ice_mass' + limValues.units = 'kg' + data_scalar.AUTHORS = AUTHOR_STR + data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP= 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total ice mass' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- limnsw ------------------ + data_scalar = Dataset(f'{output_path}/limnsw_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + limnswValues = data_scalar.createVariable('limnsw', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + limnswValues[i] = vaf_snapshot[i] * 910 + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + limnswValues.standard_name = 'land_ice_mass_not_displacing_sea_water' + limnswValues.units = 'kg' + data_scalar.AUTHORS = AUTHOR_STR + data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Mass above floatation' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- iareagr ------------------ + data_scalar = Dataset(f'{output_path}/iareagr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + iareagrValues = data_scalar.createVariable('iareagr', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + iareagrValues[i] = gia_snapshot[i] + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + iareagrValues.standard_name = 'grounded_ice_sheet_area' + iareagrValues.units = 'm2' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Grounded ice area' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- iareafl ------------------ + data_scalar = Dataset(f'{output_path}/iareafl_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + iareaflValues = data_scalar.createVariable('iareafl', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + iareaflValues[i] = fia_snapshot[i] + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + iareaflValues.standard_name = 'floating_ice_shelf_area' + iareaflValues.units = 'm2' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Floating ice area' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendacabf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendacabf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendacabfValues = data_scalar.createVariable('tendacabf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendacabfValues[i] = smb_avg[i] / 31536000.0 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendacabfValues.standard_name = 'tendency_of_land_ice_mass_due_to_surface_mass_balance' + tendacabfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total SMB flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlibmassbf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlibmassbf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlibmassbfValues = data_scalar.createVariable('tendlibmassbf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlibmassbfValues[i] = bmb_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlibmassbfValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance ' + tendlibmassbfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total BMB flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlibmassbffl: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlibmassbffl_AIS_DOE_MALI_{exp}.nc', 'w', + format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlibmassbfflValues = data_scalar.createVariable('tendlibmassbffl', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlibmassbfflValues[i] = bmbFlt_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlibmassbfflValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' + tendlibmassbfflValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total BMB flux beneath floating ice' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlicalvf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlicalvf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlicalvfValues = data_scalar.createVariable('tendlicalvf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlicalvfValues[i] = -cfx_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlicalvfValues.standard_name = 'tendency_of_land_ice_mass_due_to_calving' + tendlicalvfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total calving flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlifmassbf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlifmassbf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlicalvfValues = data_scalar.createVariable('tendlifmassbf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlicalvfValues[i] = -cfmfx_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlicalvfValues.standard_name = 'tendency_of_land_ice_mass_due_to_calving_and_ice_front_melting' + tendlicalvfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total calving and ice front melting flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendligroundf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendligroundf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendligroundfValues = data_scalar.createVariable('tendligroundf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendligroundfValues[i] = gfx_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendligroundfValues.standard_name = 'tendency_of_grounded_ice_mass' + tendligroundfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total grounding line flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + data.close() diff --git a/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py b/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py new file mode 100755 index 000000000..a524db62e --- /dev/null +++ b/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python + +""" +This script copies a restart file of a MALI simulation +and re-calculates missing state variables for a +missing time level and writes them to an updated restart file. +""" + +import argparse +import os +import shutil +import xarray as xr +import numpy as np + + +def main(): + parser = argparse.ArgumentParser( + description='process MALI outputs for the ISMIP6' + 'submission') + parser.add_argument("-f", "--file", dest="file_in", + required=True, + help="restart file to be read in") + parser.add_argument("-o", "--output_file", dest="file_out", + required=True, + help="output file name") + parser.add_argument("-p", "--output_file_path", + dest="output_path") + + args = parser.parse_args() + + # read in a restart file that needs to be re-written + if args.file_in is None: + print("--- restart file is not provided. Aborting... ---") + else: + print("\n--- Reading in the restart file ---") + + file_in = xr.open_dataset(args.file_in, decode_times=False, decode_cf=False) + + # get needed info from restart file + cellMask = file_in['cellMask'][:, :] + thickness = file_in['thickness'][:,:] + bedTopography = file_in['bedTopography'][:,:] + sfcAirTemp = file_in['surfaceAirTemperature'][:,:] + uReconstructX = file_in['uReconstructX'][:,:,:] + uReconstructY = file_in['uReconstructY'][:,:,:] + layerThicknessFractions = file_in['layerThicknessFractions'] + nTime = file_in.dims['Time'] + nCells = file_in.dims['nCells'] + nVertLevels = file_in.dims['nVertLevels'] + + # xtime needs some massaging for xarray not to mangle it + xtime = file_in['xtime'] + xtimeStr = xtime.data.tobytes().decode() # convert to str + xtime2 = xr.DataArray(np.array([xtimeStr], dtype = np.dtype(('S', 64))), dims = ['Time']) # convert back to char array + # followed example here: https://github.com/pydata/xarray/issues/3407 + + floating_iceMask = (cellMask[:, :] & 4) // 4 + seaLevel = 0.0 + rhoi = 910.0 + rhoo = 1028.0 + + print(f'nTime={nTime}, nCells={nCells}') + + layerInterfaceFractions = np.zeros(nVertLevels+1, dtype=float) + lowerSfc = np.zeros([nTime, nCells], dtype=float) + upperSfc = np.zeros([nTime, nCells], dtype=float) + sfcTemp = np.zeros([nTime, nCells], dtype=float) + xvelmean = np.zeros([nTime, nCells], dtype=float) + yvelmean = np.zeros([nTime, nCells], dtype=float) + # the following need to be in the file so ncrcat will work but processing won't use + # values, so can leave as zeros + surfaceSpeed = np.zeros([nTime, nCells], dtype=float) + vonMisesStress = np.zeros([nTime, nCells], dtype=float) + deltat = np.zeros([nTime,], dtype=float) + daysSinceStart = np.zeros([nTime,], dtype=float) + + print("\n--- calculating the missing state variables ---") + + # layerInterfaceFractions are the fraction associated with each interface + layerInterfaceFractions[0] = 0.5 * layerThicknessFractions[0] + for k in range(1, nVertLevels): + layerInterfaceFractions[k] = 0.5 * (layerThicknessFractions[k-1] + + layerThicknessFractions[k]) + layerInterfaceFractions[nVertLevels] = 0.5 * layerThicknessFractions[nVertLevels-1] + print("layerThicknessFractions:", layerThicknessFractions[:].data) + print("layerInterfaceFractions:", layerInterfaceFractions) + + for i in range(nTime): + # calculate surface temperature (unit in Kelvin) + sfcTemp[i,:] = np.minimum(273.15, sfcAirTemp[i,:]) # 0 celsius = 273 Kelvin + print('surfaceTemperature processed') + + lowerSfc[i,:] = np.where(floating_iceMask, seaLevel - thickness[i,:] * (rhoi / rhoo), bedTopography[i,:]) + upperSfc[i,:] = lowerSfc[i,:] + thickness[i,:] + print('lower/upperSurface processed') + + xvelmean[i,:] = np.sum(uReconstructX[i,:,:] * layerInterfaceFractions[:], axis=1) + yvelmean[i,:] = np.sum(uReconstructY[i,:,:] * layerInterfaceFractions[:], axis=1) + print('x/yvelmean processed') + + # create variable dictionary of fields to include in the new file + # Note: ncrcat does not require that time-independent fields be in both + # files, so we don't need to include them in the new file. + out_data_vars = { + 'lowerSurface': (['Time', 'nCells'], lowerSfc), + 'upperSurface': (['Time', 'nCells'], upperSfc), + 'surfaceTemperature': (['Time', 'nCells'], sfcTemp), + 'xvelmean': (['Time', 'nCells'], xvelmean), + 'yvelmean': (['Time', 'nCells'], yvelmean), + 'surfaceSpeed': (['Time', 'nCells'], surfaceSpeed), + 'vonMisesStress': (['Time', 'nCells'], vonMisesStress), + 'deltat': (['Time',], deltat ), + 'daysSinceStart': (['Time',], daysSinceStart), + 'xtime': xtime2 + } + dataOut = xr.Dataset(data_vars=out_data_vars) # create xarray dataset object + dataOut.xtime.encoding.update({"char_dim_name": "StrLen"}) # another hacky thing to make xarray handle xtime correctly + # learned this from: https://github.com/pydata/xarray/issues/2895 + + print("\n--- copying over unmodified variables from the restart file ---") + for var in ['thickness', 'uReconstructX', 'uReconstructY', 'bedTopography', + 'basalTemperature', 'betaSolve', 'cellMask', 'damage']: + print(" Copying", var) + dataOut[var] = file_in[var] + + # save/write out the new file + # define the path to which the output (processed) files will be saved + if args.output_path is None: + output_path = os.getcwd() + else: + output_path = args.output_path + + if not os.path.isdir(output_path): + os.makedirs(output_path) + + print(f"file output path: {output_path}") + file_out_path = os.path.join(output_path, args.file_out) + dataOut.to_netcdf(file_out_path, mode='w', unlimited_dims=['Time']) + file_in.close() + + print("\n--- process complete! ---") + +if __name__ == "__main__": + main() From 2ed7cf9de96a2ebf399729f22bae7bd935cc0b09 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 24 Jun 2026 15:38:42 -0700 Subject: [PATCH 02/33] Refactor: rename ISMIP6 postprocessing scripts for ISMIP7 --- ...p6.py => create_mapfile_mali_to_ismip7.py} | 44 +++++----- ...mip6.py => post_process_mali_to_ismip7.py} | 88 +++++++++---------- ...es.py => process_flux_variables_ismip7.py} | 52 +++++------ ...s.py => process_state_variables_ismip7.py} | 80 ++++++++--------- .../recalculate_missing_2d_state_vars.py | 2 +- 5 files changed, 133 insertions(+), 133 deletions(-) rename landice/output_processing_li/ismip7_postprocessing/{create_mapfile_mali_to_ismip6.py => create_mapfile_mali_to_ismip7.py} (72%) rename landice/output_processing_li/ismip7_postprocessing/{post_process_mali_to_ismip6.py => post_process_mali_to_ismip7.py} (79%) rename landice/output_processing_li/ismip7_postprocessing/{process_flux_variables.py => process_flux_variables_ismip7.py} (94%) rename landice/output_processing_li/ismip7_postprocessing/{process_state_variables.py => process_state_variables_ismip7.py} (93%) diff --git a/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip6.py b/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip7.py similarity index 72% rename from landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip6.py rename to landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip7.py index b9732b227..b778cf96f 100644 --- a/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip6.py +++ b/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip7.py @@ -5,13 +5,13 @@ import xarray as xr def build_mapping_file(mali_mesh_file, - mapping_file, res_ismip6_grid, - ismip6_grid_file=None, + mapping_file, res_ismip7_grid, + ismip7_grid_file=None, method_remap=None): """ Build a mapping file if it does not exist. Mapping file is then used to remap the MALI source file to the - ISMIP6 ppolarstero grid + ISMIP7 ppolarstero grid Parameters ---------- @@ -20,13 +20,13 @@ def build_mapping_file(mali_mesh_file, mali file mapping_file : str - weights for interpolation from mali_mesh_file to ismip6_grid_file + weights for interpolation from mali_mesh_file to ismip7_grid_file - res_ismip6_grid: str - resolution of the ismip6 grid in kilometers + res_ismip7_grid: str + resolution of the ismip7 grid in kilometers - ismip6_grid_file : str, optional - The ISMIP6 file if mapping file does not exist + ismip7_grid_file : str, optional + The ISMIP7 file if mapping file does not exist method_remap : str, optional Remapping method used in building a mapping file @@ -36,30 +36,30 @@ def build_mapping_file(mali_mesh_file, print(f"Mapping file exists. Not building a new one.") return - if ismip6_grid_file is None: - raise ValueError("Mapping file does not exist. To build one, ISMIP6 " + if ismip7_grid_file is None: + raise ValueError("Mapping file does not exist. To build one, ISMIP7 " "grid file with '-f' should be provided. " "Type --help for info") if method_remap is None: method_remap = "conserve" - ismip6_scripfile = f"temp_ismip6_{res_ismip6_grid}km_scrip.nc" + ismip7_scripfile = f"temp_ismip7_{res_ismip7_grid}km_scrip.nc" mali_scripfile = "temp_mali_scrip.nc" - ismip6_projection = "ais-bedmap2" + ismip7_projection = "ais-bedmap2" - # create the ismip6 scripfile if mapping file does not exist - # this is the projection of ismip6 data for Antarctica + # create the ismip7 scripfile if mapping file does not exist + # this is the projection of ismip7 data for Antarctica print(f"Mapping file does not exist. Building one based on " f"the input/ouptut meshes") print(f"Creating temporary scripfiles " - f"for ismip6 grid and mali mesh...") + f"for ismip7 grid and mali mesh...") - # create a scripfile for ismip6 grid + # create a scripfile for ismip7 grid args = ["create_SCRIP_file_from_planar_rectangular_grid.py", - "--input", ismip6_grid_file, - "--scrip", ismip6_scripfile, - "--proj", ismip6_projection, + "--input", ismip7_grid_file, + "--scrip", ismip7_scripfile, + "--proj", ismip7_projection, "--rank", "2"] check_call(args) @@ -82,7 +82,7 @@ def build_mapping_file(mali_mesh_file, if hostname.startswith('nid'): args = (["srun", "-n", "12", "ESMF_RegridWeightGen", "-s", mali_scripfile, - "-d", ismip6_scripfile, + "-d", ismip7_scripfile, "-w", mapping_file, "-m", method_remap, "-i", "-64bit_offset", @@ -90,7 +90,7 @@ def build_mapping_file(mali_mesh_file, else: args = (["ESMF_RegridWeightGen", "-s", mali_scripfile, - "-d", ismip6_scripfile, + "-d", ismip7_scripfile, "-w", mapping_file, "-m", method_remap, "-i", "-64bit_offset", @@ -100,5 +100,5 @@ def build_mapping_file(mali_mesh_file, # remove the temporary scripfiles once the mapping file is generated print(f"Removing the temporary mesh and scripfiles...") - os.remove(ismip6_scripfile) + os.remove(ismip7_scripfile) os.remove(mali_scripfile) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip6.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py similarity index 79% rename from landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip6.py rename to landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 7bf8eb7fd..cb1c82ffc 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip6.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -2,7 +2,7 @@ """ This script processes MALI simulation outputs (both state and flux) -in the required format by the ISMIP6 experimental protocol. +in the required format by the ISMIP7 experimental protocol. The input state files (i.e., output files from MALI) need to have been concatenated to have yearly data, which can be done using 'ncrcat' command before using this script. @@ -14,20 +14,20 @@ import shutil import numpy as np from netCDF4 import Dataset -from create_mapfile_mali_to_ismip6 import build_mapping_file -from process_state_variables import generate_output_2d_state_vars, \ +from create_mapfile_mali_to_ismip7 import build_mapping_file +from process_state_variables_ismip7 import generate_output_2d_state_vars, \ process_state_vars, generate_output_1d_vars -from process_flux_variables import generate_output_2d_flux_vars, \ +from process_flux_variables_ismip7 import generate_output_2d_flux_vars, \ do_time_avg_flux_vars, clean_flux_fields_before_time_averaging def main(): parser = argparse.ArgumentParser( - description='process MALI outputs for the ISMIP6' + description='process MALI outputs for the ISMIP7' 'submission') parser.add_argument("-e", "--exp_name", dest="exp", required=True, - help="ISMIP6 experiment name (e.g., exp05") + help="ISMIP7 experiment name (e.g., exp05") parser.add_argument("-i_state", "--input_state", dest="input_file_state", required=False, help="mpas output state variables") parser.add_argument("-i_flux", "--input_flux", dest="input_file_flux", @@ -45,32 +45,32 @@ def main(): help="mali mesh name (e.g., AIS_8to30km)") parser.add_argument("--mapping_file", dest="mapping_file", required=False, - help="mapping file name from MALI mesh to ISMIP6 grid") - parser.add_argument("--ismip6_grid_file", dest="ismip6_grid_file", + help="mapping file name from MALI mesh to ISMIP7 grid") + parser.add_argument("--ismip7_grid_file", dest="ismip7_grid_file", required=True, - help="Input ismip6 mesh file.") + help="Input ismip7 mesh file.") parser.add_argument("--method", dest="method_remap", default="conserve", required=False, help="mapping method. Default='conserve'") - parser.add_argument("--res", dest="res_ismip6_grid", + parser.add_argument("--res", dest="res_ismip7_grid", required=True, - help="resolution of the ismip6 grid, (e.g. 8 for 8km res)") + help="resolution of the ismip7 grid, (e.g. 8 for 8km res)") args = parser.parse_args() - print("\n---Checking the coordinate variables of the ismip6 grid file---") - data_ismip6 = Dataset(args.ismip6_grid_file, "r") - if 'x' and 'y' in data_ismip6.variables: - ismip6_grid_file = args.ismip6_grid_file + print("\n---Checking the coordinate variables of the ismip7 grid file---") + data_ismip7 = Dataset(args.ismip7_grid_file, "r") + if 'x' and 'y' in data_ismip7.variables: + ismip7_grid_file = args.ismip7_grid_file print("'x' and 'y' coordinates exist in the file.") else: print("'x' and 'y' coordinates don't exist in the file.") - print("Creating them and a copy file of the ismip6 grid file...") - copy_ismip6_file = f"temp_{os.path.basename(args.ismip6_grid_file)}" - shutil.copy2(args.ismip6_grid_file, copy_ismip6_file) - copy_ismip6_file = Dataset(copy_ismip6_file, "r+", format="netCDF4") - nx = data_ismip6.dimensions["x"].size - ny = data_ismip6.dimensions["y"].size - dx = int(args.res_ismip6_grid)*1000 + print("Creating them and a copy file of the ismip7 grid file...") + copy_ismip7_file = f"temp_{os.path.basename(args.ismip7_grid_file)}" + shutil.copy2(args.ismip7_grid_file, copy_ismip7_file) + copy_ismip7_file = Dataset(copy_ismip7_file, "r+", format="netCDF4") + nx = data_ismip7.dimensions["x"].size + ny = data_ismip7.dimensions["y"].size + dx = int(args.res_ismip7_grid)*1000 dy = dx if (nx % 2) == 0: var_x = dx*((np.arange(-nx/2, nx/2)) + 0.5) @@ -79,8 +79,8 @@ def main(): var_x = dx*((np.arange(-(nx-1)/2, (nx+1)/2))) var_y = dy*((np.arange(-(ny-1)/2, (ny+1)/2))) - x = copy_ismip6_file.createVariable("x", "d", ("x")) - y = copy_ismip6_file.createVariable("y", "d", ("y")) + x = copy_ismip7_file.createVariable("x", "d", ("x")) + y = copy_ismip7_file.createVariable("y", "d", ("y")) for i in range(nx): x[i] = var_x[i] @@ -92,15 +92,15 @@ def main(): y.units = 'm' y.standard_name = 'y' - copy_ismip6_file.close() - ismip6_grid_file = f"temp_{os.path.basename(args.ismip6_grid_file)}" - temp_ismip6_grid_file = True + copy_ismip7_file.close() + ismip7_grid_file = f"temp_{os.path.basename(args.ismip7_grid_file)}" + temp_ismip7_grid_file = True - # check the lower left and upper right corners of the ismip6 grid + # check the lower left and upper right corners of the ismip7 grid print("Checking the grid corners...") - data_ismip6 = Dataset(ismip6_grid_file, "r") - x = data_ismip6.variables["x"] - y = data_ismip6.variables["y"] + data_ismip7 = Dataset(ismip7_grid_file, "r") + x = data_ismip7.variables["x"] + y = data_ismip7.variables["y"] if not x[0] == -3040000 or not y[0] == -3040000: raise ValueError(f"The lower left corner values must be at " f"-3040000m and -3040000m. But the values are at " @@ -114,7 +114,7 @@ def main(): f"provided for '--res' matches with the resolution of " f"the MALI output files. ") else: - print(f"Grid corners are as ismip6-required: " + print(f"Grid corners are as ismip7-required: " f"lower right corner values at {x[0]}m and {y[0]}m, and " f"upper right corner values at {x[-1]}m and {y[-1]}m") @@ -134,13 +134,13 @@ def main(): method_remap = args.method_remap mapping_file = f"map_{args.mali_mesh_name}_to_"\ - f"ismip6_{args.res_ismip6_grid}km_{method_remap}.nc" + f"ismip7_{args.res_ismip7_grid}km_{method_remap}.nc" print(f"Creating new mapping file." f"Mapping method used: {method_remap}") build_mapping_file(args.input_file_grid, mapping_file, - args.res_ismip6_grid, ismip6_grid_file, + args.res_ismip7_grid, ismip7_grid_file, method_remap) print("---Processing remapping file complete---\n") @@ -159,12 +159,12 @@ def main(): else: print("\n---Processing state file---") # state variables processing part - # process (add and rename) state vars as requested by the ISMIP6 protocol + # process (add and rename) state vars as requested by the ISMIP7 protocol print("Calculating needed state file adjustments.") tmp_file = "tmp_state.nc" process_state_vars(args.input_file_state, tmp_file) - # remap data from the MALI unstructured mesh to the ISMIP6 polarstereo grid + # remap data from the MALI unstructured mesh to the ISMIP7 polarstereo grid processed_and_remapped_file_state = f'processed_and_remapped_' \ f'{os.path.basename(args.input_file_state)}' @@ -176,10 +176,10 @@ def main(): "-P", "mpas"] check_call(command) - # write out 2D state output files in the ismip6-required format - print("Writing processed and remapped state fields to ISMIP6 file format.") + # write out 2D state output files in the ismip7-required format + print("Writing processed and remapped state fields to ISMIP7 file format.") generate_output_2d_state_vars(processed_and_remapped_file_state, - ismip6_grid_file, + ismip7_grid_file, args.exp, output_path) os.remove(tmp_file) @@ -208,7 +208,7 @@ def main(): tmp_file1 = "flux_time_avg.nc" do_time_avg_flux_vars(tmp_file_translate, tmp_file1) - # remap data from the MALI unstructured mesh to the ISMIP6 P-S grid + # remap data from the MALI unstructured mesh to the ISMIP7 P-S grid processed_file_flux = f'processed_' \ f'{os.path.basename(args.input_file_flux)}' command = ["ncremap", @@ -218,9 +218,9 @@ def main(): "-P", "mpas"] check_call(command) - # write out the output files in the ismip6-required format + # write out the output files in the ismip7-required format generate_output_2d_flux_vars(processed_file_flux, - ismip6_grid_file, + ismip7_grid_file, args.exp, output_path) cleanUp = True @@ -228,8 +228,8 @@ def main(): os.remove(tmp_file_translate) os.remove(tmp_file1) os.remove(processed_file_flux) - if temp_ismip6_grid_file: - os.remove(ismip6_grid_file) + if temp_ismip7_grid_file: + os.remove(ismip7_grid_file) print("---Processing flux file complete---\n") print("---All processing complete---") diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py similarity index 94% rename from landice/output_processing_li/ismip7_postprocessing/process_flux_variables.py rename to landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index a528ac642..34c1a9181 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -1,6 +1,6 @@ """ This script has functions that are needed to post-process and write flux -output variables from ISMIP6 simulations. +output variables from ISMIP7 simulations. """ from netCDF4 import Dataset @@ -153,7 +153,7 @@ def do_time_avg_flux_vars(input_file, output_file): def clean_flux_fields_before_time_averaging(file_input, file_mesh, file_output): """ - Convert the MALI output field calvingThickness to the ISMIP6 variable + Convert the MALI output field calvingThickness to the ISMIP7 variable licalvf and apply bounds checking on BMB, where some crazy values occasionally occur. """ @@ -349,8 +349,8 @@ def clean_flux_fields_before_time_averaging(file_input, file_mesh, calvingFluxArray[t,bdyIndices] += thresholdFlux[t,bdyIndices] data['calvingFlux'] = calvingFluxArray # Note: thresholdFlux was already added in above - data['thresholdFlux'] = thresholdFlux # this is just written for diagnostic purposes. It's not actually sent to ISMIP6. - data['faceMeltAndCalvingFlux'] = faceMeltFluxArray + calvingFluxArray # ismip6 only wants the combined fields for face-melt + data['thresholdFlux'] = thresholdFlux # this is just written for diagnostic purposes. It's not actually sent to ISMIP7. + data['faceMeltAndCalvingFlux'] = faceMeltFluxArray + calvingFluxArray # ismip7 only wants the combined fields for face-melt print("===done calving flux processing!===") if debug_face_melt_flux: print('debug_face_melt_flux is True, so I assume you want a breakpoint' + @@ -359,25 +359,25 @@ def clean_flux_fields_before_time_averaging(file_input, file_mesh, data.to_netcdf(file_output) # copy of the input file with new vars added data.close() -def write_netcdf_2d_flux_vars(mali_var_name, ismip6_var_name, var_std_name, +def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, var_units, var_varname, remapped_mali_flux_file, - ismip6_grid_file, exp, output_path): + ismip7_grid_file, exp, output_path): """ mali_var_name: variable name on MALI side - ismip6_var_name: variable name required by ISMIP6 + ismip7_var_name: variable name required by ISMIP7 var_std_name: standard variable name var_units: variable units var_varname: variable variable name - remapped_mali_flux_file: mali flux file remapped on the ISMIP6 grid - ismip6_grid_file: original ISMIP6 file + remapped_mali_flux_file: mali flux file remapped on the ISMIP7 grid + ismip7_grid_file: original ISMIP7 file exp: experiment name output_path: output path to which the output files will be saved """ - data_ismip6 = Dataset(ismip6_grid_file, 'r') - var_x = data_ismip6.variables['x'][:] - var_y = data_ismip6.variables['y'][:] + data_ismip7 = Dataset(ismip7_grid_file, 'r') + var_x = data_ismip7.variables['x'][:] + var_y = data_ismip7.variables['y'][:] data = Dataset(remapped_mali_flux_file, 'r') data.set_auto_mask(False) @@ -394,14 +394,14 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip6_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip6_var_name}_AIS_DOE_MALI_{exp}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('bnds', 2) timebndsValues = dataOut.createVariable('time_bnds', 'd', ('time', 'bnds')) dataOut.createDimension('x', lonN) dataOut.createDimension('y', latN) - dataValues = dataOut.createVariable(ismip6_var_name, 'd', + dataValues = dataOut.createVariable(ismip7_var_name, 'd', ('time', 'y', 'x'), fill_value=np.NAN) xValues = dataOut.createVariable('x', 'd', ('x')) yValues = dataOut.createVariable('y', 'd', ('y')) @@ -450,13 +450,13 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip6_var_name, var_std_name, def generate_output_2d_flux_vars(file_remapped_mali_flux, - ismip6_grid_file, exp, output_path): + ismip7_grid_file, exp, output_path): """ file_remapped_mali_flux: flux output file on mali mesh remapped - onto the ismip6 grid - ismip6 grid - ismip6_grid_file: ismip6 original file - exp: ISMIP6 experiment name + onto the ismip7 grid + ismip7 grid + ismip7_grid_file: ismip7 original file + exp: ISMIP7 experiment name output_path: path to which the final output files are saved """ @@ -466,7 +466,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'land_ice_surface_specific_mass_balance_flux', 'kg m-2 s-1', 'Surface mass balance flux', file_remapped_mali_flux, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- libmassbffl ------------------ write_netcdf_2d_flux_vars('libmassbffl', 'libmassbffl', @@ -474,7 +474,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Basal mass balance flux beneath floating ice', file_remapped_mali_flux, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- libmassbfgr ------------------ write_netcdf_2d_flux_vars('libmassbfgr', 'libmassbfgr', @@ -482,7 +482,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Basal mass balance flux beneath grounded ice', file_remapped_mali_flux, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- dlithkdt ------------------ write_netcdf_2d_flux_vars('dHdt', 'dlithkdt', @@ -490,7 +490,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'm s-1', 'Ice thickness imbalance', file_remapped_mali_flux, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- licalvf ------------------ write_netcdf_2d_flux_vars('calvingFlux', 'licalvf', @@ -498,7 +498,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Calving flux', file_remapped_mali_flux, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- lifmassbf ------------------ # Note: facemelting and calving flux are combined above @@ -507,7 +507,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Ice front melt and calving flux', file_remapped_mali_flux, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- ligroundf ------------------ write_netcdf_2d_flux_vars('fluxAcrossGroundingLineOnCells', 'ligroundf', @@ -515,4 +515,4 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Grounding line flux', file_remapped_mali_flux, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py similarity index 93% rename from landice/output_processing_li/ismip7_postprocessing/process_state_variables.py rename to landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 6e8388d3f..bb4541995 100755 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -1,6 +1,6 @@ """ This script has functions that are needed to post-process and write state -output variables from ISMIP6 simulations. +output variables from ISMIP7 simulations. The input files (i.e., MALI output files) need to have been concatenated to have yearly data, which can be done using 'ncrcat' command before using this script. @@ -58,24 +58,24 @@ def process_state_vars(inputfile_state, tmp_file): inputfile_state_vars.close() -def write_netcdf_2d_state_vars(mali_var_name, ismip6_var_name, var_std_name, +def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, var_units, var_varname, remapped_mali_outputfile, - ismip6_grid_file, exp, output_path): + ismip7_grid_file, exp, output_path): """ mali_var_name: variable name on MALI side - ismip6_var_name: variable name required by ISMIP6 + ismip7_var_name: variable name required by ISMIP7 var_std_name: standard variable name var_units: variable units var_varname: variable variable name - remapped_mali_outputfile: mali state file remapped on the ISMIP6 grid - ismip6_grid_file: original ISMIP6 file + remapped_mali_outputfile: mali state file remapped on the ISMIP7 grid + ismip7_grid_file: original ISMIP7 file exp: experiment name output_path: output path to which the output files will be saved """ - data_ismip6 = Dataset(ismip6_grid_file, 'r') - var_x = data_ismip6.variables['x'][:] - var_y = data_ismip6.variables['y'][:] + data_ismip7 = Dataset(ismip7_grid_file, 'r') + var_x = data_ismip7.variables['x'][:] + var_y = data_ismip7.variables['y'][:] data = Dataset(remapped_mali_outputfile, 'r') data.set_auto_mask(False) @@ -89,12 +89,12 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip6_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip6_var_name}_AIS_DOE_MALI_{exp}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('x', lonN) dataOut.createDimension('y', latN) - dataValues = dataOut.createVariable(ismip6_var_name, 'd', + dataValues = dataOut.createVariable(ismip7_var_name, 'd', ('time', 'y', 'x'), fill_value=np.NAN) xValues = dataOut.createVariable('x', 'd', ('x')) yValues = dataOut.createVariable('y', 'd', ('y')) @@ -104,14 +104,14 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip6_var_name, var_std_name, DATE_STR = date.today().strftime("%d-%b-%Y") for i in range(timeSteps): - if ismip6_var_name == 'sftgif': + if ismip7_var_name == 'sftgif': dataValues[i, :, :] = var_mali[i, :, :] else: - if ismip6_var_name == 'litempbotgr': + if ismip7_var_name == 'litempbotgr': mask = var_sftgrf[i, :, :] - elif ismip6_var_name == 'litempbotfl': + elif ismip7_var_name == 'litempbotfl': mask = var_sftflf[i, :, :] - elif ismip6_var_name == 'topg': + elif ismip7_var_name == 'topg': mask = np.ones(var_mali.shape[1:]) # don't mask topg else: mask = var_sftgif[i, :, :] @@ -146,12 +146,12 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip6_var_name, var_std_name, def generate_output_2d_state_vars(file_remapped_mali_state, - ismip6_grid_file, exp, output_path): + ismip7_grid_file, exp, output_path): """ file_remapped_mali_state: output files on mali mesh remapped - on the ismip6 grid - ismip6_grid_file: ismip6 original file - exp: ISMIP6 experiment name + on the ismip7 grid + ismip7_grid_file: ismip7 original file + exp: ISMIP7 experiment name output_path: path to which the final output files are saved """ @@ -160,25 +160,25 @@ def generate_output_2d_state_vars(file_remapped_mali_state, write_netcdf_2d_state_vars('thickness','lithk', 'land_ice_thickness', 'm', 'Ice thickness', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- orog ------------------ write_netcdf_2d_state_vars('upperSurface','orog', 'surface_altitude', 'm', 'Surface elevation', file_remapped_mali_state, - ismip6_grid_file,exp, output_path) + ismip7_grid_file,exp, output_path) # ----------- base ------------------ write_netcdf_2d_state_vars('lowerSurface','base', 'base_altitude', 'm', 'Base elevation', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- topg ------------------ write_netcdf_2d_state_vars('bedTopography','topg', 'bedrock_altitude', 'm', 'Bedrock elevation', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- hfgeoubed------------------ # Note: even though this is a flux variable, we are taking a snapshot of it @@ -186,7 +186,7 @@ def generate_output_2d_state_vars(file_remapped_mali_state, # write_netcdf_2d_state_vars('basalHeatFlux', 'hfgeoubed', # 'upward_geothermal_heat_flux_in_land_ice', # 'W m-2', 'Geothermal heat flux', - # file_remapped_mali_state, ismip6_grid_file, + # file_remapped_mali_state, ismip7_grid_file, # exp, output_path) # ----------- xvelsurf ------------------ @@ -194,31 +194,31 @@ def generate_output_2d_state_vars(file_remapped_mali_state, 'land_ice_surface_x_velocity', 'm s-1', 'Surface velocity in x', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # -----------yxvelsurf ------------------ write_netcdf_2d_state_vars('uReconstructY_sfc', 'yvelsurf', 'land_ice_surface_y_velocity', 'm s-1', 'Surface velocity in x', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- xvelbase ------------------ write_netcdf_2d_state_vars('uReconstructX_base', 'xvelbase', 'land_ice_basal_x_velocity', 'm s-1', 'Basal velocity in x', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- yvelbase ------------------ write_netcdf_2d_state_vars('uReconstructY_base', 'yvelbase', 'land_ice_basal_y_velocity', 'm s-1', 'Basal velocity in y', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- zvelsurf & zvelbase ------------------ - # ISMIP6 requires these variables, but MALI does not output them. + # ISMIP7 requires these variables, but MALI does not output them. # So, we are not processing/writing these variables out. # ----------- xvelmean ------------------ @@ -226,70 +226,70 @@ def generate_output_2d_state_vars(file_remapped_mali_state, 'land_ice_vertical_mean_x_velocity', 'm s-1', 'Mean velocity in x', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- yvelmean ------------------ write_netcdf_2d_state_vars('yvelmean', 'yvelmean', 'land_ice_vertical_mean_y_velocity', 'm s-1', 'Mean velocity in y', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- litemptop ------------------ write_netcdf_2d_state_vars('surfaceTemperature', 'litemptop', 'temperature_at_top_of_ice_sheet_model', 'K', 'Surface temperature', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- litempbotgr ------------------ write_netcdf_2d_state_vars('litempbotgr', 'litempbotgr', 'temperature_at_base_of_ice_sheet_model', 'K', 'Basal temperature beneath grounded ice sheet', file_remapped_mali_state, - ismip6_grid_file,exp, output_path) + ismip7_grid_file,exp, output_path) # ----------- litempbotfl ------------------ write_netcdf_2d_state_vars('litempbotfl', 'litempbotfl', 'temperature_at_base_of_ice_sheet_model', 'K', 'Basal temperature beneath floating ice shelf', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- strbasemag ------------------ write_netcdf_2d_state_vars('strbasemag', 'strbasemag', 'land_ice_basal_drag ', 'Pa', 'Basal drag', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- sftgif ------------------ write_netcdf_2d_state_vars('sftgif','sftgif', 'land_ice_area_fraction', '1', 'Land ice area fraction', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- sftgrf ------------------ write_netcdf_2d_state_vars('sftgrf', 'sftgrf', 'grounded_ice_sheet_area_fraction', '1', 'Grounded ice sheet area fraction', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) # ----------- sftflf ------------------ write_netcdf_2d_state_vars('sftflf','sftflf', 'floating_ice_shelf_area_fraction', '1', 'Floating ice shelf area fraction', file_remapped_mali_state, - ismip6_grid_file, exp, output_path) + ismip7_grid_file, exp, output_path) def generate_output_1d_vars(global_stats_file, exp, output_path=None): """ This code processes both state and flux 1D variables global_stats_file: MALI globalStats.nc output file - exp: ISMIP6 experiment number + exp: ISMIP7 experiment number output_path: """ @@ -325,7 +325,7 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): # For flux fields, the years is the calendar year being averaged over, # e.g., flux year 2000 is the average between Jan. 1, 2000, and Jan. 1, 2001. # Note this year convention differs from the first column in table in A2.3.2 at - # https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP6-Projections2300-Antarctica#A2.3.3_Table_A1:_Variable_request_for_ISMIP6 + # https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP7-Projections2300-Antarctica#A2.3.3_Table_A1:_Variable_request_for_ISMIP6 # but that year indexing convention ultimately doesn't matter because the # time coordinates in these files uses units of days since a reference date, # and it does not use a year indexing convention at all. diff --git a/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py b/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py index a524db62e..315715677 100755 --- a/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py +++ b/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py @@ -15,7 +15,7 @@ def main(): parser = argparse.ArgumentParser( - description='process MALI outputs for the ISMIP6' + description='process MALI outputs for the ISMIP7' 'submission') parser.add_argument("-f", "--file", dest="file_in", required=True, From 5b6459188ad1a0e3805aad541b22a93bdf590330 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 24 Jun 2026 17:52:01 -0700 Subject: [PATCH 03/33] Use time-averaged variables for 2D flux outputs This change follows updates to MALI in which flux outputs are averaged online over each output interval. Also update flux-variable metadata to match the ISMIP7 variable request (https://docs.google.com/spreadsheets/d/1yWDxk8pRwwp3gvRT6j0lsWA_5hAkAXqZdLGtgpnLSNE/edit?gid=2113789754#gid=2113789754) and stop calling the legacy flux cleaning/time-averaging functions from the main script: post_process_mali_to_ismip7.py --- .../post_process_mali_to_ismip7.py | 15 ++------------ .../process_flux_variables_ismip7.py | 20 +++++++++---------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index cb1c82ffc..d36c1e206 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -17,9 +17,7 @@ from create_mapfile_mali_to_ismip7 import build_mapping_file from process_state_variables_ismip7 import generate_output_2d_state_vars, \ process_state_vars, generate_output_1d_vars -from process_flux_variables_ismip7 import generate_output_2d_flux_vars, \ - do_time_avg_flux_vars, clean_flux_fields_before_time_averaging - +from process_flux_variables_ismip7 import generate_output_2d_flux_vars def main(): parser = argparse.ArgumentParser( @@ -201,18 +199,11 @@ def main(): else: print("\n---Processing flux file---") - print("Adjusting flux fields that need modification before time averaging.") - tmp_file_translate = "flux_translated.nc" - clean_flux_fields_before_time_averaging(args.input_file_flux, args.input_file_grid, tmp_file_translate) - # take time (yearly) average for the flux variables - tmp_file1 = "flux_time_avg.nc" - do_time_avg_flux_vars(tmp_file_translate, tmp_file1) - # remap data from the MALI unstructured mesh to the ISMIP7 P-S grid processed_file_flux = f'processed_' \ f'{os.path.basename(args.input_file_flux)}' command = ["ncremap", - "-i", tmp_file1, + "-i", args.input_file_flux, "-o", processed_file_flux, "-m", mapping_file, "-P", "mpas"] @@ -225,8 +216,6 @@ def main(): cleanUp = True if cleanUp: - os.remove(tmp_file_translate) - os.remove(tmp_file1) os.remove(processed_file_flux) if temp_ismip7_grid_file: os.remove(ismip7_grid_file) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index 34c1a9181..ee722f103 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -462,14 +462,14 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, print("Writing 2d flux variables") # ----------- acabf ------------------ - write_netcdf_2d_flux_vars('sfcMassBalApplied', 'acabf', + write_netcdf_2d_flux_vars('avgSMBFlux', 'acabf', 'land_ice_surface_specific_mass_balance_flux', 'kg m-2 s-1', 'Surface mass balance flux', file_remapped_mali_flux, ismip7_grid_file, exp, output_path) # ----------- libmassbffl ------------------ - write_netcdf_2d_flux_vars('libmassbffl', 'libmassbffl', + write_netcdf_2d_flux_vars('avgFloatingBMBFlux', 'libmassbffl', 'land_ice_basal_specific_mass_balance_flux', 'kg m-2 s-1', 'Basal mass balance flux beneath floating ice', @@ -477,7 +477,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, ismip7_grid_file, exp, output_path) # ----------- libmassbfgr ------------------ - write_netcdf_2d_flux_vars('libmassbfgr', 'libmassbfgr', + write_netcdf_2d_flux_vars('avgGroundedBMBFlux', 'libmassbfgr', 'land_ice_basal_specific_mass_balance_flux', 'kg m-2 s-1', 'Basal mass balance flux beneath grounded ice', @@ -485,7 +485,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, ismip7_grid_file, exp, output_path) # ----------- dlithkdt ------------------ - write_netcdf_2d_flux_vars('dHdt', 'dlithkdt', + write_netcdf_2d_flux_vars('avgDhdt', 'dlithkdt', 'tendency_of_land_ice_thickness', 'm s-1', 'Ice thickness imbalance', @@ -493,7 +493,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, ismip7_grid_file, exp, output_path) # ----------- licalvf ------------------ - write_netcdf_2d_flux_vars('calvingFlux', 'licalvf', + write_netcdf_2d_flux_vars('avgCalvingFlux', 'licalvf', 'land_ice_specific_mass_flux_due_to_calving', 'kg m-2 s-1', 'Calving flux', @@ -502,16 +502,16 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, # ----------- lifmassbf ------------------ # Note: facemelting and calving flux are combined above - write_netcdf_2d_flux_vars('faceMeltAndCalvingFlux', 'lifmassbf', - 'land_ice_specific_mass_flux_due_to_calving_and_ice_front_melting', + write_netcdf_2d_flux_vars('avgFaceMeltFlux', 'lifmassbf', + 'TBD by ISMIP7', 'kg m-2 s-1', - 'Ice front melt and calving flux', + 'Ice front melt flux', file_remapped_mali_flux, ismip7_grid_file, exp, output_path) # ----------- ligroundf ------------------ - write_netcdf_2d_flux_vars('fluxAcrossGroundingLineOnCells', 'ligroundf', - 'land_ice_specific_mass_flux_at_grounding_line', + write_netcdf_2d_flux_vars('avgGroundingLineFlux', 'ligroundf', + 'TBD by ISMIP7', 'kg m-2 s-1', 'Grounding line flux', file_remapped_mali_flux, From 1bb5cc81ca749a875d1ced6eab9441198149dbee Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 24 Jun 2026 18:06:08 -0700 Subject: [PATCH 04/33] Remove legacy flux preprocessing functions MALI now provides interval-averaged flux outputs directly, so these legacy ISMIP6 preprocessing steps are no longer needed. --- .../process_flux_variables_ismip7.py | 347 ------------------ 1 file changed, 347 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index ee722f103..e327c3bac 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -12,353 +12,6 @@ import warnings -def do_time_avg_flux_vars(input_file, output_file): - """ - input_file: MALI simulation flux file that has the all time levels - output_file: file with time-averaged fluxes - """ - print("Starting time averaging of flux variables") - dataIn = xr.open_dataset(input_file, decode_cf=False) # need decode_cf=False to prevent xarray from reading daysSinceStart as a timedelta type. - if 'units' in dataIn.daysSinceStart.attrs: # make have been removed in a previous step, so check if it exists - del dataIn.daysSinceStart.attrs['units'] # need this line to prevent xarray from reading daysSinceStart as a timedelta type. - - time = dataIn.dims['Time'] - nCells = dataIn.dims['nCells'] - xtimeIn = dataIn['xtime'][:].values - #print(xtimeIn) - xtime = [] - for i in range(time): - xtime.append(xtimeIn[i].tostring().decode('utf-8').strip().strip('\x00')) - #print(xtime) - deltat = dataIn['deltat'][:] - daysSinceStart = dataIn['daysSinceStart'][:] - cellMask = dataIn['cellMask'][:,:] - sfcMassBal = dataIn['sfcMassBalApplied'][:, :] - floatingBasalMassBalApplied = dataIn['floatingBasalMassBalApplied'][:, :] - groundedBasalMassBalApplied = dataIn['groundedBasalMassBalApplied'][:, :] - dHdt = dataIn['dHdt'][:,:] / (3600.0 * 24.0 * 365.0) # convert units to m/s - glFlux = dataIn['fluxAcrossGroundingLineOnCells'][:, :] - calvingFlux = dataIn['calvingFlux'][:, :] - faceMeltAndCalvingFlux = dataIn['faceMeltAndCalvingFlux'][:, :] - - iceMask = (cellMask[:, :] & 2) / 2 # grounded: dynamic ice - - # Figure out some timekeeping stuff - using netCDF4 b/c xarray is a nightmare - fin = Dataset(input_file, 'r') - simulationStartTime = fin.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') - fin.close() - simulationStartDate = simulationStartTime.split("_")[0] - if simulationStartDate[5:10] != '01-01': - sys.exit("Error: simulationStartTime for flux file is not on Jan. 1.") - refYear = int(simulationStartDate[0:4]) - startYr = refYear + np.floor(daysSinceStart[0] / 365.0) # using floor here because we might not have output at jan 1, but we'll definitely have at least one time level per year - finalYr = refYear + daysSinceStart[-1] / 365.0 - if (daysSinceStart[-1] / 365.0 != daysSinceStart[-1] // 365): - sys.exit(f"Error: final time of flux output file is not on Jan. 1.: daysSinceStart={daysSinceStart[-1]}, xtime={xtime[-1]}" ) - print(f"simulationStartTime={simulationStartTime}; simulationStartDate={simulationStartDate}; refYear={refYear}") - print(f"start year={startYr}; final year={finalYr}") - - # get an array of years that are not duplicative - decYears = refYear + daysSinceStart/365.0 - #years = np.floor(decYears - 1.0e-10) # this is the "owning" year; Jan 1 belongs to the previous year, so offset decYears by small amount - #years[0] = int(xtime[0].decode("utf-8")[0:4]) - ##years = np.trim_zeros(years) - years = np.arange(startYr, finalYr) # we don't want the final year in the time array as a year to process - it's actually the end point of the previous year - - timeBndsMin = np.ones((len(years),)) * 1.0e36 - timeBndsMax = np.ones((len(years),)) * -1.0e36 - - avgSmb = np.zeros((len(years), nCells)) * np.nan - avgCF = np.zeros((len(years), nCells)) * np.nan - avgCFandFM = np.zeros((len(years), nCells)) * np.nan - avgBmbfl = np.zeros((len(years), nCells)) * np.nan - avgBmbgr = np.zeros((len(years), nCells)) * np.nan - avgDHdt = np.zeros((len(years), nCells)) * np.nan - avgGF = np.zeros((len(years), nCells)) * np.nan - maxIceMask = np.zeros((len(years), nCells), dtype=np.int) * np.nan - - print(" begin looping over years") - for j in range(len(years)): - # we want time bounds to span the full year - timeBndsMin[j] = (years[j] - refYear) * 365.0 - timeBndsMax[j] = (years[j]+1.0 - refYear) * 365.0 - print(f" year index: {j}, year={years[j]}; timeBindsMin={timeBndsMin[j]}, timeBndsMax={timeBndsMax[j]}") - sumYearSmb = 0 - sumYearBmb = 0 - sumYearDHdt = 0 - sumYearCF = 0 - sumYearCFandFM = 0 - sumYearGF = 0 - sumYearBHF = 0 - sumYearTime = 0 - sumYearBmbfl = 0 - sumYearBmbgr = 0 - sumIceMask = 0 - - timeBndMin = 1.0e36 - timeBndMax = -1.0e36 - for i in range(time): - - if decYears[i] > years[j] and decYears[i] <= years[j]+1.0: - sumYearSmb = sumYearSmb + sfcMassBal[i, :] * deltat[i] - sumYearBmbfl = sumYearBmbfl + floatingBasalMassBalApplied[i, :] * deltat[i] - sumYearBmbgr = sumYearBmbgr + groundedBasalMassBalApplied[i, :] * deltat[i] - sumYearDHdt = sumYearDHdt + dHdt[i, :] * deltat[i] - sumYearCF = sumYearCF + calvingFlux[i,:] * deltat[i] - sumYearCFandFM = sumYearCFandFM + faceMeltAndCalvingFlux[i,:] * deltat[i] - sumYearGF = sumYearGF + glFlux[i, :] * deltat[i] - sumYearTime = sumYearTime + deltat[i] - - sumIceMask = sumIceMask + iceMask[i,:] - - print(f" year={years[j]}, decYears={decYears[i]}, daysSinceStart={daysSinceStart[j]}, xtime={xtime[i]}") - - - avgSmb[j,:] = sumYearSmb / sumYearTime - avgBmbfl[j,:] = sumYearBmbfl / sumYearTime - avgBmbgr[j,:] = sumYearBmbgr / sumYearTime - avgDHdt[j,:] = sumYearDHdt / sumYearTime - avgCF[j,:] = sumYearCF / sumYearTime - avgCFandFM[j,:] = sumYearCFandFM / sumYearTime - avgGF[j,:] = sumYearGF / sumYearTime - maxIceMask[j,:] = (sumIceMask>0) # Get mask for anywhere that had ice during this year - - - print(" write time averaged values") - - print(f"avg shape={avgSmb.shape}, time shape={timeBndsMin.shape}") - out_data_vars = { - 'sfcMassBalApplied': (['Time', 'nCells'], avgSmb), - 'libmassbffl': (['Time', 'nCells'], avgBmbfl), - 'libmassbfgr': (['Time', 'nCells'], avgBmbgr), - 'dHdt': (['Time', 'nCells'], avgDHdt), - 'fluxAcrossGroundingLineOnCells': (['Time', 'nCells'], avgGF), - 'calvingFlux': (['Time', 'nCells'], avgCF), - 'faceMeltAndCalvingFlux': (['Time', 'nCells'], avgCFandFM), - 'iceMask': (['Time', 'nCells'], maxIceMask), - 'timeBndsMin': (['Time'], timeBndsMin), - 'timeBndsMax': (['Time'], timeBndsMax), - 'simulationStartTime': dataIn['simulationStartTime'] - } - out_coords = { - 'Time': (['Time'], (timeBndsMin+timeBndsMax)/2.0) - } - - - dataOut = xr.Dataset(data_vars=out_data_vars, coords=out_coords) - dataOut.to_netcdf(output_file, mode='w') - dataIn.close() - - -def clean_flux_fields_before_time_averaging(file_input, file_mesh, - file_output): - """ - Convert the MALI output field calvingThickness to the ISMIP7 variable - licalvf and apply bounds checking on BMB, where some crazy values occasionally occur. - """ - - debug_face_melt_flux = False - data = xr.open_dataset(file_input, decode_cf=False) # need decode_cf=False to prevent xarray from reading daysSinceStart as a timedelta type. - if 'units' in data.daysSinceStart.attrs: - del data.daysSinceStart.attrs['units'] # need this line to prevent xarray from reading daysSinceStart as a timedelta type. - time = data.dims['Time'] - nCells = data.dims['nCells'] - nEdgesOnCell = data['nEdgesOnCell'][:].values - edgesOnCell = data['edgesOnCell'][:].values - cellsOnCell = data['cellsOnCell'][:].values - dvEdge = data['dvEdge'][:].values - areaCell = data['areaCell'][:].values - xCell = data['xCell'][:].values - yCell = data['yCell'][:].values - deltat = data['deltat'][:].values - thickness = data['thickness'][:].values - surfaceSpeed = data['surfaceSpeed'][:].values - if 'bedTopography' in data: - bedTopography = data['bedTopography'][:].values - print('bedTopography field found; using bedTopography at all time levels.') - else: - data_mesh = xr.open_dataset(file_mesh) - bedTopography = data_mesh['bedTopography'][:].values - print('No bedTopography field found; using bedTopography from mesh file.') - if 'calvingThicknessFromThreshold' in data: - calvingThicknessFromThreshold = data['calvingThicknessFromThreshold'][:, :].values - else: - print('WARNING: No calvingThicknessFromThreshold field found; creating a field populated with zeros.') - calvingThicknessFromThreshold = thickness.copy() * 0.0 - - rho_i = 910.0 - - print("===starting cleaning floatingBasalMassBalApplied===") - # We've encountered a few enormous BMB values. Until we solve where that is coming from, - # set them to something reasonable. - floatingBasalMassBalApplied = data['floatingBasalMassBalApplied'][:, :].values - for t in range(1, time): - if t%20 == 0: - print(f" Time: {t+1} / {time}") - # Set large negative BMB values (1 m/s) to the equivalent of the thickness from the previous time step - # (Commented line here picked up too many places) - #ind = np.nonzero((-floatingBasalMassBalApplied[t,:]/rho_i*deltat[t] > thickness[t-1,:]) * (thickness[t-1,:]>0.0))[0] - ind = np.nonzero(floatingBasalMassBalApplied[t,:]/rho_i < -1.0)[0] - if len(ind) > 0: - print(f"Fixing {len(ind)} cells with large negative floating BMB values.", ind) - floatingBasalMassBalApplied[t, ind] = -thickness[t-1, ind] / deltat[t] * rho_i - - ind = np.nonzero(floatingBasalMassBalApplied[t,:]/rho_i > 1.0)[0] - if len(ind) > 0: - print(f"Fixing {len(ind)} cells with large positive floating BMB values.", ind) - ind2 = np.nonzero(floatingBasalMassBalApplied[t,:]/rho_i <= 1.0)[0] - maxGoodBMB = floatingBasalMassBalApplied[t, ind2].max() - floatingBasalMassBalApplied[t, ind] = maxGoodBMB - - print("===done cleaning floatingBasalMassBalApplied===") - - assert time == len(deltat) - - calvingVelocity = data['calvingVelocity'][:, :].values - - # create and initialize a new data array for calvingFluxArray - calvingFluxArray = data['calvingVelocity'].copy() * 0.0 - thresholdFlux = data['calvingVelocity'].copy() * 0.0 - calvingThickness = data['calvingThickness'][:, :].values - print("===starting facemelt flux processing===") - - # create and initialize a new data array for faceMeltFluxArray - # (copied from calving code below) - # Some runs won't have this output field, so assume if field is not present - # that facemelting was not enabled - faceMeltFluxArray = data['calvingVelocity'].copy() * 0.0 - if 'faceMeltSpeed' in data: - faceMeltSpeed = data['faceMeltSpeed'][:, :].values - # faceMeltSpeed is defined below the water line, but face-melting is - # applied to the full ice thickness, so the effective speed is - # averaged over the full thickness from the previous time step. - # Note that this calculation assumes that bedTopography is constant in time, - # that config_sea_level = 0, and that faceMeltSpeed is only valid for - # grounded cells, i.e., that bedTopography and lowerSurface are equivalent - # (which is currently the case). - - faceMeltingThickness = data['faceMeltingThickness'][:, :].values - faceMeltSpeedVertAvg = faceMeltingThickness.copy() * 0.0 - # Fields for validation and debugging - if debug_face_melt_flux: - deltat_array = np.tile(deltat, (np.shape(faceMeltSpeed)[1],1)).transpose() - # Cleaned field for debugging and validation - faceMeltingThicknessCleaned = faceMeltingThickness.copy() - for t in range(time): - if t%20 == 0: - print(f" Time: {t+1} / {time}") - - if 'bedTopography' in data: - bed = bedTopography[t,:] # have value per time level - else: - bed = bedTopography[0,:] # just have a single value - - prev_t = max(t-1, 0) # ensure that index_cf never uses thickness from last (-1) time step - index_cf = np.where((faceMeltingThickness[t, :] > 0.0) * (bed[:] < 0.0) * - (faceMeltingThickness[t, :] != thickness[prev_t, :]) * - (thickness[prev_t, :] > 0.))[0] - for i in index_cf: - # faceMeltSpeed is calculated for ice below water line, but needs to be aplied - # to full ice thickness, so we need a vertically averaged speed. Also ensure that - # the vertically averaged speed is never > faceMeltSpeed due to small ice thickness. - # This may be slightly inaccurate on the very first time step. - faceMeltSpeedVertAvg[t,i] = faceMeltSpeed[t, i] * np.abs(bed[i] / thickness[prev_t, i]) - faceMeltSpeedVertAvg[t,i] = min(faceMeltSpeedVertAvg[t,i], faceMeltSpeed[t, i]) - # Use this cell if it has nonzero faceMeltingThickness because faceMeltSpeed - # is defined everywhere, but only applied on grounded ice - if faceMeltingThickness[t,i] > 0.0: - faceMeltFluxArray[t,i] = faceMeltSpeedVertAvg[t,i] * rho_i # convert to proper units - # Push mass removed from stranded non-dynamic cells into calving - index_stranded_cell_cleanup = np.where(faceMeltingThickness[t, :] == thickness[prev_t, :])[0] - calvingThicknessFromThreshold[t, index_stranded_cell_cleanup] += faceMeltingThickness[t, index_stranded_cell_cleanup] - if debug_face_melt_flux: - faceMeltingThicknessCleaned[t, index_stranded_cell_cleanup] -= faceMeltingThickness[t, index_stranded_cell_cleanup] - # This is just for debugging and validation - print("===done facemelt flux processing!===") - - print("===starting the calving flux processing===") - - for t in range(time): - if t%20 == 0: - print(f" Time: {t+1} / {time}") - - if 'bedTopography' in data: - bed = bedTopography[t,:] # have value per time level - else: - bed = bedTopography[0,:] # just have a single value - - index_cf = np.where((calvingVelocity[t, :] > 0.0) * (bed[:] < 0.0))[0] - for i in index_cf: - ne = nEdgesOnCell[i] - for j in range(ne): - neighborCellId = cellsOnCell[i, j] - 1 - # Use this cell if it has a neighbor with zero calvingVelocity that is below sea level - if calvingVelocity[t,neighborCellId] == 0.0 and bed[neighborCellId] < 0.0: - calvingFluxArray[t,i] = calvingVelocity[t,i] * rho_i # convert to proper units - continue # no need to keep searching the neighbors of this cell - - - # we may need to add on threshold calving too - if 'calvingThicknessFromThreshold' in data: - index_cf = np.where(calvingThicknessFromThreshold[t, :] > 0.0)[0] - else: - index_cf = [] - if len(index_cf) > 0: - thresholdBoundary = np.zeros((nCells,), 'i') - thresholdBoundaryAssignedVolume = np.zeros((nCells,)) - thresholdBoundarySummedThickness = np.zeros((nCells,)) - thresholdBoundaryContributors = np.zeros((nCells,)) - thresholdBoundaryLength = np.zeros((nCells,)) - thresholdSpeed = np.zeros((nCells,)) - # First make list of boundary cells calved - for i in index_cf: - ne = nEdgesOnCell[i] - for j in range(ne): - neighborCellId = cellsOnCell[i, j] - 1 - if thickness[t,neighborCellId] > 0.0 and bed[neighborCellId] < 0.0 and calvingThicknessFromThreshold[t,neighborCellId] == 0.0: - thresholdBoundary[i] = 1 - thresholdBoundaryLength[i] += dvEdge[edgesOnCell[i,j]-1] - bdyIndices = np.where(thresholdBoundary == 1)[0] - print(f"Found {len(index_cf)} cells with threshold calving at time {t}; {len(bdyIndices)} are boundary cells.") - if len(bdyIndices) == 0: - print(f"0 boundary cells were found; skipping to next time step") - continue - # Now loop over all threshold cells and assign their volume to the nearest boundary cell - for i in index_cf: - if thresholdBoundary[i] == 1: - ownerIdx = i # often the cell is its own owner, so check before doing the more expensive search - else: - ownerIdx = bdyIndices[np.argmin((xCell[i]-xCell[bdyIndices])**2 + (yCell[i]-yCell[bdyIndices])**2)] - thresholdBoundaryAssignedVolume[ownerIdx] += calvingThicknessFromThreshold[t,i] * areaCell[i] - thresholdBoundarySummedThickness[ownerIdx] += calvingThicknessFromThreshold[t,i] - thresholdBoundaryContributors[ownerIdx] += 1 - #print(thresholdBoundaryAssignedVolume.sum(), (calvingThicknessFromThreshold[t,:]*areaCell[:]).sum()) - diff = np.absolute(thresholdBoundaryAssignedVolume.sum() - (calvingThicknessFromThreshold[t,:]*areaCell[:]).sum()) - if diff < 1.0: - warnings.warn(f"Difference between assigned `thresholdBoundaryAssignedVolume` value and " - f"`calvingThicknessFromThreshold` threshold value is less than 1: {diff} < 1.0") - #for i in bdyIndices: - #print(f"length={thresholdBoundaryLength[i]}, vol={thresholdBoundaryAssignedVolume[i]}, sumthk={thresholdBoundarySummedThickness[i]}, num={thresholdBoundaryContributors[i]}, meanthk={thresholdBoundarySummedThickness[i]/thresholdBoundaryContributors[i]}") - # Finally calculate licalvf for each boundary cell and add to whatever was already there - thresholdSpeed[bdyIndices] = thresholdBoundaryAssignedVolume[bdyIndices] / \ - (thresholdBoundarySummedThickness[bdyIndices] / thresholdBoundaryContributors[bdyIndices] * \ - thresholdBoundaryLength[bdyIndices]) / \ - deltat[t] # units of m/s - # Our estimated threshold speed is really a retreat speed. So to get calving speed, add on the advective speed - thresholdFlux[t, bdyIndices] += (thresholdSpeed[bdyIndices] + surfaceSpeed[t,bdyIndices]) * rho_i - calvingFluxArray[t,bdyIndices] += thresholdFlux[t,bdyIndices] - - data['calvingFlux'] = calvingFluxArray # Note: thresholdFlux was already added in above - data['thresholdFlux'] = thresholdFlux # this is just written for diagnostic purposes. It's not actually sent to ISMIP7. - data['faceMeltAndCalvingFlux'] = faceMeltFluxArray + calvingFluxArray # ismip7 only wants the combined fields for face-melt - print("===done calving flux processing!===") - if debug_face_melt_flux: - print('debug_face_melt_flux is True, so I assume you want a breakpoint' + - ' to check fluxes. Just type continue when you want to proceed.') - breakpoint() - data.to_netcdf(file_output) # copy of the input file with new vars added - data.close() - def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, var_units, var_varname, remapped_mali_flux_file, ismip7_grid_file, exp, output_path): From d7eb53ec8c42b89b52f917c242a08064d1b141fd Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 24 Jun 2026 20:46:24 -0700 Subject: [PATCH 05/33] Implement mandatory ISMIP7 scalar flux outputs Add the grounded basal mass balance scalar output required by ISMIP7, update the ice-front melting scalar to use face melting only, and remove the legacy total basal mass balance scalar output. --- .../process_state_variables_ismip7.py | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index bb4541995..f95c2f4bc 100755 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -359,16 +359,16 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): # read in flux variables over which yearly average will be taken smb = data.variables['totalSfcMassBal'][:] - bmb = data.variables['totalBasalMassBal'][:] + bmbGr = data.variables['totalGroundedBasalMassBal'][:] # clean out some garbage values we can't account for - ind = np.nonzero(bmb>1.0e18)[0] + ind = np.nonzero(bmbGr > 1.0e18)[0] if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalBasalMassBal>1.0e18") - bmb[ind] = np.nan - ind = np.nonzero(bmb<-1.0e18)[0] + print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal>1.0e18") + bmbGr[ind] = np.nan + ind = np.nonzero(bmbGr < -1.0e18)[0] if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalBasalMassBal<-1.0e18") - bmb[ind] = np.nan + print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal<-1.0e18") + bmbGr[ind] = np.nan bmbFlt = data.variables['totalFloatingBasalMassBal'][:] # clean out some garbage values we can't account for ind = np.nonzero(bmbFlt>1.0e18)[0] @@ -391,10 +391,10 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): fia_snapshot = np.zeros(nt_state) * np.nan days_snapshot = np.zeros(nt_state) * np.nan smb_avg = np.zeros(nt_flux) * np.nan - bmb_avg = np.zeros(nt_flux) * np.nan + bmbGr_avg = np.zeros(nt_flux) * np.nan bmbFlt_avg = np.zeros(nt_flux) * np.nan cfx_avg = np.zeros(nt_flux) * np.nan - cfmfx_avg = np.zeros(nt_flux) * np.nan + fmfx_avg = np.zeros(nt_flux) * np.nan gfx_avg = np.zeros(nt_flux) * np.nan days_min = np.zeros(nt_flux) * np.nan days_max = np.zeros(nt_flux) * np.nan @@ -417,19 +417,19 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): ind_avg = np.where(np.logical_and(decYears > years_flux[i], decYears <= (years_flux[i] + 1.0)))[0] smbi = smb[ind_avg] - bmbi = bmb[ind_avg] + bmbGri = bmbGr[ind_avg] bmbFlti = bmbFlt[ind_avg] cfxi = cfx[ind_avg] - cfmfxi = cfx[ind_avg] + fmfx[ind_avg] + fmfxi = fmfx[ind_avg] gfxi = gfx[ind_avg] dti = dt[ind_avg] # take the average of the flux variables smb_avg[i] = np.nansum(smbi * dti) / np.nansum(dti) - bmb_avg[i] = np.nansum(bmbi * dti) / np.nansum(dti) + bmbGr_avg[i] = np.nansum(bmbGri * dti) / np.nansum(dti) bmbFlt_avg[i] = np.nansum(bmbFlti * dti) / np.nansum(dti) cfx_avg[i] = np.nansum(cfxi * dti) / np.nansum(dti) - cfmfx_avg[i] = np.nansum(cfmfxi * dti) / np.nansum(dti) + fmfx_avg[i] = np.nansum(fmfxi * dti) / np.nansum(dti) gfx_avg[i] = np.nansum(gfxi * dti) / np.nansum(dti) days_min[i] = (years_flux[i] - refYear) * 365.0 days_max[i] = (years_flux[i] + 1.0 - refYear) * 365.0 @@ -546,15 +546,15 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): data_scalar.DATE = DATE_STR data_scalar.close() - # -------------- tendlibmassbf: this is a flux var - data_scalar = Dataset(f'{output_path}/tendlibmassbf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + # -------------- tendlibmassbfgr: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlibmassbfgr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') data_scalar.createDimension('time', nt_flux) - tendlibmassbfValues = data_scalar.createVariable('tendlibmassbf', 'd', ('time')) + tendlibmassbfgrValues = data_scalar.createVariable('tendlibmassbfgr', 'd', ('time')) timeValues = data_scalar.createVariable('time', 'd', ('time')) data_scalar.createDimension('bnds', 2) timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) for i in range(nt_flux): - tendlibmassbfValues[i] = bmb_avg[i] / 31536000 + tendlibmassbfgrValues[i] = bmbGr_avg[i] / 31536000.0 timeValues[i] = (days_min[i] + days_max[i]) / 2.0 timebndsValues[i, 0] = days_min[i] timebndsValues[i, 1] = days_max[i] @@ -562,12 +562,12 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): timeValues.calendar = 'noleap' timeValues.standard_name = 'time' timeValues.long_name = 'time' - tendlibmassbfValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance ' - tendlibmassbfValues.units = 'kg s-1' + tendlibmassbfgrValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' + tendlibmassbfgrValues.units = 'kg s-1' data_scalar.AUTHORS= AUTHOR_STR data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total BMB flux' + data_scalar.VARIABLE = 'Total BMB flux beneath grounded ice' data_scalar.DATE = DATE_STR data_scalar.close() @@ -623,14 +623,16 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): data_scalar.close() # -------------- tendlifmassbf: this is a flux var + # In ISMIP6, this variable used to be 'Total calving and ice front melting flux' + # In ISMIP7, it represents 'Total ice front melting flux' only, without calving flux data_scalar = Dataset(f'{output_path}/tendlifmassbf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') data_scalar.createDimension('time', nt_flux) - tendlicalvfValues = data_scalar.createVariable('tendlifmassbf', 'd', ('time')) + tendlifmassbfValues = data_scalar.createVariable('tendlifmassbf', 'd', ('time')) timeValues = data_scalar.createVariable('time', 'd', ('time')) data_scalar.createDimension('bnds', 2) timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) for i in range(nt_flux): - tendlicalvfValues[i] = -cfmfx_avg[i] / 31536000 + tendlifmassbfValues[i] = -fmfx_avg[i] / 31536000 timeValues[i] = (days_min[i] + days_max[i]) / 2.0 timebndsValues[i, 0] = days_min[i] timebndsValues[i, 1] = days_max[i] @@ -638,12 +640,12 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): timeValues.calendar = 'noleap' timeValues.standard_name = 'time' timeValues.long_name = 'time' - tendlicalvfValues.standard_name = 'tendency_of_land_ice_mass_due_to_calving_and_ice_front_melting' - tendlicalvfValues.units = 'kg s-1' + tendlifmassbfValues.standard_name = 'tendency_of_land_ice_mass_due_to_ice_front_melting' + tendlifmassbfValues.units = 'kg s-1' data_scalar.AUTHORS= AUTHOR_STR data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total calving and ice front melting flux' + data_scalar.VARIABLE = 'Total ice front melting flux' data_scalar.DATE = DATE_STR data_scalar.close() From b4982d31b2a3bcafa535c863ed245764b0239857 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 09:19:52 -0700 Subject: [PATCH 06/33] Simplify args for map files * eliminate mapping file name and mali grid name and replace with pre-generated map file name * add option to reuse existing map file --- .../post_process_mali_to_ismip7.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index d36c1e206..b304b4638 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -12,6 +12,7 @@ from subprocess import check_call import os import shutil +from datetime import datetime import numpy as np from netCDF4 import Dataset from create_mapfile_mali_to_ismip7 import build_mapping_file @@ -38,14 +39,10 @@ def main(): required=False, help="path to which the final output files" " will be saved") - parser.add_argument("--mali_mesh_name", dest="mali_mesh_name", - required=True, - help="mali mesh name (e.g., AIS_8to30km)") - parser.add_argument("--mapping_file", dest="mapping_file", + parser.add_argument("--reuse_mapping_file", dest="reuse_mapping_file", required=False, - help="mapping file name from MALI mesh to ISMIP7 grid") + help="existing mapping file name to reuse") parser.add_argument("--ismip7_grid_file", dest="ismip7_grid_file", - required=True, help="Input ismip7 mesh file.") parser.add_argument("--method", dest="method_remap", default="conserve", required=False, @@ -57,6 +54,7 @@ def main(): print("\n---Checking the coordinate variables of the ismip7 grid file---") data_ismip7 = Dataset(args.ismip7_grid_file, "r") + temp_ismip7_grid_file = False if 'x' and 'y' in data_ismip7.variables: ismip7_grid_file = args.ismip7_grid_file print("'x' and 'y' coordinates exist in the file.") @@ -119,20 +117,23 @@ def main(): print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process if not args.input_file_state is None or not args.input_file_flux is None: - # check the mapping method and existence of the mapping file + # Check mapping method and either reuse an existing map or create a new one. # Note: the function 'building_mapping_file' requires the mpas mesh tool # script 'create_SCRIP_file_from_planar_rectangular_grid.py' - if os.path.exists(args.mapping_file): - print(f"Mapping file exists.") - mapping_file = args.mapping_file + method_remap = args.method_remap + if args.reuse_mapping_file is not None: + if not os.path.exists(args.reuse_mapping_file): + raise FileNotFoundError(f"Mapping file to reuse not found: " + f"{args.reuse_mapping_file}") + print(f"Reusing existing mapping file: {args.reuse_mapping_file}") + mapping_file = args.reuse_mapping_file else: - if args.method_remap is None: - method_remap = "conserve" - else: - method_remap = args.method_remap + if args.input_file_grid is None: + raise ValueError("--input_mesh is required when creating a new " + "mapping file.") - mapping_file = f"map_{args.mali_mesh_name}_to_"\ - f"ismip7_{args.res_ismip7_grid}km_{method_remap}.nc" + created_at = datetime.now().strftime("%Y%m%dT%H%M%S") + mapping_file = f"mapping_mali_to_ismip7.{method_remap}.{created_at}.nc" print(f"Creating new mapping file." f"Mapping method used: {method_remap}") From d5b6c98fb1ab72910629d064cbb57b3d5cac13e4 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 09:27:14 -0700 Subject: [PATCH 07/33] Create combined module for mapping and grid checks * Renames create_mapfile_mali_to_ismip7.py to grid_and_mapping.py * moves grid check stuff from main script to the new module --- ..._mali_to_ismip7.py => grid_and_mapping.py} | 86 +++++++++++++++++++ .../post_process_mali_to_ismip7.py | 68 +-------------- ...ile_from_planar_rectangular_grid_latlon.py | 74 ++++++++++++++++ 3 files changed, 163 insertions(+), 65 deletions(-) rename landice/output_processing_li/ismip7_postprocessing/{create_mapfile_mali_to_ismip7.py => grid_and_mapping.py} (50%) create mode 100755 mesh_tools/create_SCRIP_files/create_SCRIP_file_from_planar_rectangular_grid_latlon.py diff --git a/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py similarity index 50% rename from landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip7.py rename to landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py index b778cf96f..8ec2c165b 100644 --- a/landice/output_processing_li/ismip7_postprocessing/create_mapfile_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py @@ -1,8 +1,94 @@ from mpas_tools.scrip.from_mpas import scrip_from_mpas from subprocess import check_call import os +import shutil import netCDF4 import xarray as xr +import numpy as np + + +def prepare_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): + """ + Ensure the ISMIP7 grid file has 'x' and 'y' coordinate variables and that + its corners match the ISMIP7-required extents. If 'x'/'y' are absent a + temporary copy with those variables added is created. + + Parameters + ---------- + ismip7_grid_file_path : str + Path to the ISMIP7 grid file supplied by the user. + res_ismip7_grid : str + Resolution of the ISMIP7 grid in kilometres (e.g. '8'). + + Returns + ------- + ismip7_grid_file : str + Path to the grid file to use (may be a temporary copy). + temp_file_created : bool + True if a temporary copy was created and should be removed after use. + """ + print("\n---Checking the coordinate variables of the ismip7 grid file---") + data_ismip7 = netCDF4.Dataset(ismip7_grid_file_path, "r") + temp_file_created = False + + if 'x' in data_ismip7.variables and 'y' in data_ismip7.variables: + ismip7_grid_file = ismip7_grid_file_path + print("'x' and 'y' coordinates exist in the file.") + else: + print("'x' and 'y' coordinates don't exist in the file.") + print("Creating them and a copy file of the ismip7 grid file...") + tmp_path = f"temp_{os.path.basename(ismip7_grid_file_path)}" + shutil.copy2(ismip7_grid_file_path, tmp_path) + copy_ds = netCDF4.Dataset(tmp_path, "r+", format="netCDF4") + nx = data_ismip7.dimensions["x"].size + ny = data_ismip7.dimensions["y"].size + dx = int(res_ismip7_grid) * 1000 + dy = dx + if (nx % 2) == 0: + var_x = dx * ((np.arange(-nx / 2, nx / 2)) + 0.5) + var_y = dy * ((np.arange(-ny / 2, ny / 2)) + 0.5) + else: + var_x = dx * (np.arange(-(nx - 1) / 2, (nx + 1) / 2)) + var_y = dy * (np.arange(-(ny - 1) / 2, (ny + 1) / 2)) + x = copy_ds.createVariable("x", "d", ("x",)) + y = copy_ds.createVariable("y", "d", ("y",)) + x[:] = var_x + y[:] = var_y + x.units = 'm' + x.standard_name = 'x' + y.units = 'm' + y.standard_name = 'y' + copy_ds.close() + ismip7_grid_file = tmp_path + temp_file_created = True + + data_ismip7.close() + + print("Checking the grid corners...") + check_ds = netCDF4.Dataset(ismip7_grid_file, "r") + x = check_ds.variables["x"] + y = check_ds.variables["y"] + if not x[0] == -3040000 or not y[0] == -3040000: + raise ValueError( + f"The lower left corner values must be at " + f"-3040000m and -3040000m. But the values are at " + f"{x[0]}m and {y[0]}m. Check the value you " + f"provided for '--res' matches with the resolution of " + f"the MALI output files. ") + elif not x[-1] == 3040000 or not y[-1] == 3040000: + raise ValueError( + f"The upper right corner values must be at " + f"3040000m and 3040000m. But the values are at " + f"{x[-1]}m and {y[-1]}m. Check the value you " + f"provided for '--res' matches with the resolution of " + f"the MALI output files. ") + else: + print(f"Grid corners are as ismip7-required: " + f"lower left corner values at {x[0]}m and {y[0]}m, and " + f"upper right corner values at {x[-1]}m and {y[-1]}m") + check_ds.close() + + return ismip7_grid_file, temp_file_created def build_mapping_file(mali_mesh_file, mapping_file, res_ismip7_grid, diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index b304b4638..268b9bc58 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -11,11 +11,8 @@ import argparse from subprocess import check_call import os -import shutil from datetime import datetime -import numpy as np -from netCDF4 import Dataset -from create_mapfile_mali_to_ismip7 import build_mapping_file +from grid_and_mapping import build_mapping_file, prepare_ismip7_grid_file from process_state_variables_ismip7 import generate_output_2d_state_vars, \ process_state_vars, generate_output_1d_vars from process_flux_variables_ismip7 import generate_output_2d_flux_vars @@ -52,67 +49,8 @@ def main(): help="resolution of the ismip7 grid, (e.g. 8 for 8km res)") args = parser.parse_args() - print("\n---Checking the coordinate variables of the ismip7 grid file---") - data_ismip7 = Dataset(args.ismip7_grid_file, "r") - temp_ismip7_grid_file = False - if 'x' and 'y' in data_ismip7.variables: - ismip7_grid_file = args.ismip7_grid_file - print("'x' and 'y' coordinates exist in the file.") - else: - print("'x' and 'y' coordinates don't exist in the file.") - print("Creating them and a copy file of the ismip7 grid file...") - copy_ismip7_file = f"temp_{os.path.basename(args.ismip7_grid_file)}" - shutil.copy2(args.ismip7_grid_file, copy_ismip7_file) - copy_ismip7_file = Dataset(copy_ismip7_file, "r+", format="netCDF4") - nx = data_ismip7.dimensions["x"].size - ny = data_ismip7.dimensions["y"].size - dx = int(args.res_ismip7_grid)*1000 - dy = dx - if (nx % 2) == 0: - var_x = dx*((np.arange(-nx/2, nx/2)) + 0.5) - var_y = dy*((np.arange(-ny/2, ny/2)) + 0.5) - else: - var_x = dx*((np.arange(-(nx-1)/2, (nx+1)/2))) - var_y = dy*((np.arange(-(ny-1)/2, (ny+1)/2))) - - x = copy_ismip7_file.createVariable("x", "d", ("x")) - y = copy_ismip7_file.createVariable("y", "d", ("y")) - - for i in range(nx): - x[i] = var_x[i] - for i in range(ny): - y[i] = var_y[i] - - x.units = 'm' - x.standard_name = 'x' - y.units = 'm' - y.standard_name = 'y' - - copy_ismip7_file.close() - ismip7_grid_file = f"temp_{os.path.basename(args.ismip7_grid_file)}" - temp_ismip7_grid_file = True - - # check the lower left and upper right corners of the ismip7 grid - print("Checking the grid corners...") - data_ismip7 = Dataset(ismip7_grid_file, "r") - x = data_ismip7.variables["x"] - y = data_ismip7.variables["y"] - if not x[0] == -3040000 or not y[0] == -3040000: - raise ValueError(f"The lower left corner values must be at " - f"-3040000m and -3040000m. But the values are at " - f"{x[0]}m and {y[0]}m. Check the value you " - f"provided for '--res' matches with the resolution of " - f"the MALI output files. ") - elif not x[-1] == 3040000 or not y[-1] == 3040000: - raise ValueError(f"The upper right corner values must be at " - f"3040000m and 3040000m. But the values are at " - f"{x[-1]}m and {y[-1]}m. Check the value you " - f"provided for '--res' matches with the resolution of " - f"the MALI output files. ") - else: - print(f"Grid corners are as ismip7-required: " - f"lower right corner values at {x[0]}m and {y[0]}m, and " - f"upper right corner values at {x[-1]}m and {y[-1]}m") + ismip7_grid_file, temp_ismip7_grid_file = prepare_ismip7_grid_file( + args.ismip7_grid_file, args.res_ismip7_grid) print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process diff --git a/mesh_tools/create_SCRIP_files/create_SCRIP_file_from_planar_rectangular_grid_latlon.py b/mesh_tools/create_SCRIP_files/create_SCRIP_file_from_planar_rectangular_grid_latlon.py new file mode 100755 index 000000000..0f4627ab9 --- /dev/null +++ b/mesh_tools/create_SCRIP_files/create_SCRIP_file_from_planar_rectangular_grid_latlon.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# Create a SCRIP file from a planar rectanfular mesh. +# See for details: http://www.earthsystemmodeling.org/esmf_releases/public/ESMF_5_2_0rp1/ESMF_refdoc/node3.html#SECTION03024000000000000000 + +import sys +import netCDF4 +import numpy as np +from optparse import OptionParser + + +print ("== Gathering information. (Invoke with --help for more details. All arguments are optional)") +parser = OptionParser() +parser.description = "This script takes an MPAS grid file and generates a SCRIP grid file." +parser.add_option("-i", "--input", dest="inputFile", help="input grid file name used as input.", default="input.nc", metavar="FILENAME") +parser.add_option("-s", "--scrip", dest="scripFile", help="SCRIP grid file to output.", default="scrip.nc", metavar="FILENAME") + +for option in parser.option_list: + if option.default != ("NO", "DEFAULT"): + option.help += (" " if option.help else "") + "[default: %default]" +options, args = parser.parse_args() + +if not options.inputFile: + sys.exit('Error: Data input grid file is required. Specify with -c command line argument.') +if not options.scripFile: + sys.exit('Error: SCRIP output grid file is required. Specify with -s command line argument.') +print ('') # make a space in stdout before further output + +# =================================== + +fin = netCDF4.Dataset(options.inputFile, 'r') +fout = netCDF4.Dataset(options.scripFile, 'w') # This will clobber existing files + +# Get info from input file +nx = len(fin.dimensions['x']) +ny = len(fin.dimensions['y']) + +# Write to output file +# Dimensions +fout.createDimension("grid_size", nx * ny) +fout.createDimension("grid_corners", 4 ) + +print('grid rank is 2') +fout.createDimension("grid_rank", 2) + +# Variables +grid_center_lat = fout.createVariable('grid_center_lat', 'f8', ('grid_size',)) +grid_center_lat.units = 'degrees' +grid_center_lon = fout.createVariable('grid_center_lon', 'f8', ('grid_size',)) +grid_center_lon.units = 'degrees' +grid_corner_lat = fout.createVariable('grid_corner_lat', 'f8', ('grid_size', 'grid_corners')) +grid_corner_lat.units = 'degrees' +grid_corner_lon = fout.createVariable('grid_corner_lon', 'f8', ('grid_size', 'grid_corners')) +grid_corner_lon.units = 'degrees' +grid_imask = fout.createVariable('grid_imask', 'i4', ('grid_size',)) +grid_imask.units = 'unitless' +grid_dims = fout.createVariable('grid_dims', 'i4', ('grid_rank',)) + + +grid_center_lat[:] = fin.variables['lat'][:].flatten() +grid_center_lon[:] = fin.variables['lon'][:].flatten() + +lat_bnds = fin.variables['lat_bnds'][:] +grid_corner_lat[:] = lat_bnds.reshape(-1, lat_bnds.shape[-1]) +lon_bnds = fin.variables['lon_bnds'][:] +grid_corner_lon[:] = lon_bnds.reshape(-1, lon_bnds.shape[-1]) + +grid_imask[:] = 1 # For now, assume we don't want to mask anything out - but eventually may want to exclude certain cells from the input mesh during interpolation + +# set the grid dimension based on the grid rank +grid_dims[:] = [nx , ny] + +fin.close() +fout.close() +print('scrip file generation complete') From 8f042b114e779be62f5c0a52c4047690e8230636 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 09:33:59 -0700 Subject: [PATCH 08/33] Update grid check * if grid is invalid, err instead of generating grid * rename function to "check" --- .../ismip7_postprocessing/grid_and_mapping.py | 43 +++---------------- .../post_process_mali_to_ismip7.py | 13 +++--- 2 files changed, 12 insertions(+), 44 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py index 8ec2c165b..4a1822a82 100644 --- a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py +++ b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py @@ -1,13 +1,11 @@ from mpas_tools.scrip.from_mpas import scrip_from_mpas from subprocess import check_call import os -import shutil import netCDF4 import xarray as xr -import numpy as np -def prepare_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): +def check_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): """ Ensure the ISMIP7 grid file has 'x' and 'y' coordinate variables and that its corners match the ISMIP7-required extents. If 'x'/'y' are absent a @@ -22,45 +20,20 @@ def prepare_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): Returns ------- - ismip7_grid_file : str - Path to the grid file to use (may be a temporary copy). - temp_file_created : bool - True if a temporary copy was created and should be removed after use. + None """ print("\n---Checking the coordinate variables of the ismip7 grid file---") data_ismip7 = netCDF4.Dataset(ismip7_grid_file_path, "r") - temp_file_created = False if 'x' in data_ismip7.variables and 'y' in data_ismip7.variables: ismip7_grid_file = ismip7_grid_file_path print("'x' and 'y' coordinates exist in the file.") else: - print("'x' and 'y' coordinates don't exist in the file.") - print("Creating them and a copy file of the ismip7 grid file...") - tmp_path = f"temp_{os.path.basename(ismip7_grid_file_path)}" - shutil.copy2(ismip7_grid_file_path, tmp_path) - copy_ds = netCDF4.Dataset(tmp_path, "r+", format="netCDF4") - nx = data_ismip7.dimensions["x"].size - ny = data_ismip7.dimensions["y"].size - dx = int(res_ismip7_grid) * 1000 - dy = dx - if (nx % 2) == 0: - var_x = dx * ((np.arange(-nx / 2, nx / 2)) + 0.5) - var_y = dy * ((np.arange(-ny / 2, ny / 2)) + 0.5) - else: - var_x = dx * (np.arange(-(nx - 1) / 2, (nx + 1) / 2)) - var_y = dy * (np.arange(-(ny - 1) / 2, (ny + 1) / 2)) - x = copy_ds.createVariable("x", "d", ("x",)) - y = copy_ds.createVariable("y", "d", ("y",)) - x[:] = var_x - y[:] = var_y - x.units = 'm' - x.standard_name = 'x' - y.units = 'm' - y.standard_name = 'y' - copy_ds.close() - ismip7_grid_file = tmp_path - temp_file_created = True + data_ismip7.close() + raise ValueError( + f"'x' and/or 'y' coordinate variables are missing from the " + f"ISMIP7 grid file '{ismip7_grid_file_path}'. Please provide a " + f"grid file that includes 'x' and 'y' coordinate variables.") data_ismip7.close() @@ -88,8 +61,6 @@ def prepare_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): f"upper right corner values at {x[-1]}m and {y[-1]}m") check_ds.close() - return ismip7_grid_file, temp_file_created - def build_mapping_file(mali_mesh_file, mapping_file, res_ismip7_grid, ismip7_grid_file=None, diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 268b9bc58..60ab682df 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -12,7 +12,7 @@ from subprocess import check_call import os from datetime import datetime -from grid_and_mapping import build_mapping_file, prepare_ismip7_grid_file +from grid_and_mapping import build_mapping_file, check_ismip7_grid_file from process_state_variables_ismip7 import generate_output_2d_state_vars, \ process_state_vars, generate_output_1d_vars from process_flux_variables_ismip7 import generate_output_2d_flux_vars @@ -49,8 +49,7 @@ def main(): help="resolution of the ismip7 grid, (e.g. 8 for 8km res)") args = parser.parse_args() - ismip7_grid_file, temp_ismip7_grid_file = prepare_ismip7_grid_file( - args.ismip7_grid_file, args.res_ismip7_grid) + check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process @@ -77,7 +76,7 @@ def main(): f"Mapping method used: {method_remap}") build_mapping_file(args.input_file_grid, mapping_file, - args.res_ismip7_grid, ismip7_grid_file, + args.res_ismip7_grid, args.ismip7_grid_file, method_remap) print("---Processing remapping file complete---\n") @@ -116,7 +115,7 @@ def main(): # write out 2D state output files in the ismip7-required format print("Writing processed and remapped state fields to ISMIP7 file format.") generate_output_2d_state_vars(processed_and_remapped_file_state, - ismip7_grid_file, + args.ismip7_grid_file, args.exp, output_path) os.remove(tmp_file) @@ -150,14 +149,12 @@ def main(): # write out the output files in the ismip7-required format generate_output_2d_flux_vars(processed_file_flux, - ismip7_grid_file, + args.ismip7_grid_file, args.exp, output_path) cleanUp = True if cleanUp: os.remove(processed_file_flux) - if temp_ismip7_grid_file: - os.remove(ismip7_grid_file) print("---Processing flux file complete---\n") print("---All processing complete---") From d7cfa4b2f9aa0e90085c128aaba4f3729b1ed5cb Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 09:38:19 -0700 Subject: [PATCH 09/33] Add exp check function --- .../ismip7_postprocessing/grid_and_mapping.py | 23 +++++++++++++++++++ .../post_process_mali_to_ismip7.py | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py index 4a1822a82..f89950185 100644 --- a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py +++ b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py @@ -5,6 +5,29 @@ import xarray as xr +VALID_EXPERIMENTS = [f"C{i:03d}" for i in range(1, 12)] + + +def check_exp_name(exp): + """ + Validate the ISMIP7 experiment name. + + Parameters + ---------- + exp : str + Experiment name to validate (e.g. 'C001'). + + Raises + ------ + ValueError + If the experiment name is not in the list of valid experiments. + """ + if exp not in VALID_EXPERIMENTS: + raise ValueError( + f"Invalid experiment name '{exp}'. " + f"Valid experiments are: {', '.join(VALID_EXPERIMENTS)}") + print(f"Experiment name '{exp}' is valid.") + def check_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): """ Ensure the ISMIP7 grid file has 'x' and 'y' coordinate variables and that diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 60ab682df..8691402f7 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -12,7 +12,8 @@ from subprocess import check_call import os from datetime import datetime -from grid_and_mapping import build_mapping_file, check_ismip7_grid_file +from grid_and_mapping import build_mapping_file, check_ismip7_grid_file, \ + check_exp_name from process_state_variables_ismip7 import generate_output_2d_state_vars, \ process_state_vars, generate_output_1d_vars from process_flux_variables_ismip7 import generate_output_2d_flux_vars @@ -49,6 +50,7 @@ def main(): help="resolution of the ismip7 grid, (e.g. 8 for 8km res)") args = parser.parse_args() + check_exp_name(args.exp) check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) print("\n---Processing remapping file---") From 05a892bf2b916f08d182a247754d98b1504c65a3 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 09:43:15 -0700 Subject: [PATCH 10/33] slight reorg of main function --- .../post_process_mali_to_ismip7.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 8691402f7..389da3498 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -92,6 +92,16 @@ def main(): if not os.path.isdir(output_path): os.makedirs(output_path) + # process 1D variables + if args.global_stats_file is None: + print("--- MALI global stats file is not provided, thus it will not be processed.") + else: + print("\n---Processing global stats file---") + generate_output_1d_vars(args.global_stats_file, args.exp, + output_path) + print("---Processing global stats file complete---\n") + + # process 2d state variables if args.input_file_state is None: print("--- MALI state file is not provided, thus it will not be processed.") else: @@ -124,16 +134,7 @@ def main(): os.remove(processed_and_remapped_file_state) print("---Processing state file complete---\n") - # write out 1D output files for both state and flux variables - if args.global_stats_file is None: - print("--- MALI global stats file is not provided, thus it will not be processed.") - else: - print("\n---Processing global stats file---") - generate_output_1d_vars(args.global_stats_file, args.exp, - output_path) - print("---Processing global stats file complete---\n") - - # process the flux variables if flux output file is given + # process 2d flux variables if args.input_file_flux is None: print("--- MALI flux file is not provided, thus it will not be processed.") else: From e6e24d8c620b8fc4b8763fd20892fed0bff92172 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 09:46:17 -0700 Subject: [PATCH 11/33] move 1d processing to its own module --- .../post_process_mali_to_ismip7.py | 3 +- .../process_1d_variables_ismip7.py | 401 ++++++++++++++++++ .../process_state_variables_ismip7.py | 392 ----------------- 3 files changed, 403 insertions(+), 393 deletions(-) create mode 100644 landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py mode change 100755 => 100644 landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 389da3498..049af6e4d 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -15,7 +15,8 @@ from grid_and_mapping import build_mapping_file, check_ismip7_grid_file, \ check_exp_name from process_state_variables_ismip7 import generate_output_2d_state_vars, \ - process_state_vars, generate_output_1d_vars + process_state_vars +from process_1d_variables_ismip7 import generate_output_1d_vars from process_flux_variables_ismip7 import generate_output_2d_flux_vars def main(): diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py new file mode 100644 index 000000000..f0f1fa75a --- /dev/null +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -0,0 +1,401 @@ +""" +Functions for processing and writing 1D (scalar time-series) output variables +for ISMIP7 submissions from MALI globalStats output. +""" + +from netCDF4 import Dataset +from datetime import date +import numpy as np +import os +import sys + +def generate_output_1d_vars(global_stats_file, exp, output_path=None): + """ + This code processes both state and flux 1D variables + global_stats_file: MALI globalStats.nc output file + exp: ISMIP7 experiment number + output_path: + """ + + if not os.path.exists(output_path): + output_path = os.getcwd() + + AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' + DATE_STR = date.today().strftime("%d-%b-%Y") + + data = Dataset(global_stats_file, 'r') + nt_in = len(data.dimensions['Time']) + xtime = data.variables['xtime'][:, :] + daysSinceStart = data.variables['daysSinceStart'][:] + dt = data.variables['deltat'][:] + simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + simulationStartDate = simulationStartTime.split("_")[0] + if simulationStartDate[5:10] != '01-01': + sys.exit("Error: simulationStartTime for globalStats file is not on Jan. 1.") + refYear = int(simulationStartDate[0:4]) + decYears = refYear + daysSinceStart/365.0 + endYr = decYears[-1] + if endYr != np.round(endYr): + sys.exit("Error: end year not an even year in globalStats file.") + + # Determine processed time levels for state and flux fields + # The historical state fields should include the initial time (Jan. 1). + # Projection state fields should not include the initial time (Jan. 1) + # of the projection because it's a restart from the historical. + # Flux fields should never use the Jan. 1 time level at the start of the + # year as part of the averaging. + # For year conventions here, for state fields, the year is the snapshot at + # the start of the year, e.g., state year 2000 means the snapshot at Jan. 1, 2000. + # For flux fields, the years is the calendar year being averaged over, + # e.g., flux year 2000 is the average between Jan. 1, 2000, and Jan. 1, 2001. + # Note this year convention differs from the first column in table in A2.3.2 at + # https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP7-Projections2300-Antarctica#A2.3.3_Table_A1:_Variable_request_for_ISMIP6 + # but that year indexing convention ultimately doesn't matter because the + # time coordinates in these files uses units of days since a reference date, + # and it does not use a year indexing convention at all. + if decYears[0] == np.round(decYears[0]): + # The initial time level will only be on an even year (Jan. 1) + # for the hist run. In that case, we want to include that initial + # even year in the state processing. We also want the state snapshot + # at the final (even) year in the output. + # The flux processing should start with the first year, which covers a + # full 12 months. We exclude the final year, which is just a Jan. 1 posting. + years_state = np.arange(decYears[0], endYr + 1) + years_flux = np.arange(decYears[0], endYr) + else: + # For projection runs, the first state snapshot we want is the first Jan. 1, + # which we be the first even year after the initial time in the file. + # For flux files, the first full year we want to process is the year of the + # first time level in the file. As with hist, we exclude the final year, + # which is just a Jan. 1 posting. + years_state = np.arange(np.ceil(decYears[0]), endYr + 1) + years_flux = np.arange(np.floor(decYears[0]), endYr) + nt_state = len(years_state) + nt_flux = len(years_flux) + print(f'For state processing, using start year={years_state[0]} and end year={years_state[-1]}.') + print(f'For flux processing, using start year={years_flux[0]} and end year={years_flux[-1]}.') + + # read in state variables + vol = data.variables['totalIceVolume'][:] + vaf = data.variables['volumeAboveFloatation'][:] + gia = data.variables['groundedIceArea'][:] + fia = data.variables['floatingIceArea'][:] + + # read in flux variables over which yearly average will be taken + smb = data.variables['totalSfcMassBal'][:] + bmbGr = data.variables['totalGroundedBasalMassBal'][:] + # clean out some garbage values we can't account for + ind = np.nonzero(bmbGr > 1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal>1.0e18") + bmbGr[ind] = np.nan + ind = np.nonzero(bmbGr < -1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal<-1.0e18") + bmbGr[ind] = np.nan + bmbFlt = data.variables['totalFloatingBasalMassBal'][:] + # clean out some garbage values we can't account for + ind = np.nonzero(bmbFlt>1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal>1.0e18") + bmbFlt[ind] = np.nan + ind = np.nonzero(bmbFlt<-1.0e18)[0] + if len(ind) > 0: + print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal<-1.0e18") + bmbFlt[ind] = np.nan + cfx = data.variables['totalCalvingFlux'][:] + fmfx = data.variables['totalFaceMeltingFlux'][:] + gfx = data.variables['groundingLineFlux'][:] + + # initialize 1D variables that will store data value on the + # January 1st of each year + vol_snapshot = np.zeros(nt_state) * np.nan + vaf_snapshot = np.zeros(nt_state) * np.nan + gia_snapshot = np.zeros(nt_state) * np.nan + fia_snapshot = np.zeros(nt_state) * np.nan + days_snapshot = np.zeros(nt_state) * np.nan + smb_avg = np.zeros(nt_flux) * np.nan + bmbGr_avg = np.zeros(nt_flux) * np.nan + bmbFlt_avg = np.zeros(nt_flux) * np.nan + cfx_avg = np.zeros(nt_flux) * np.nan + fmfx_avg = np.zeros(nt_flux) * np.nan + gfx_avg = np.zeros(nt_flux) * np.nan + days_min = np.zeros(nt_flux) * np.nan + days_max = np.zeros(nt_flux) * np.nan + + # this is for the state variables + for i in range(nt_state): + ind_snap = np.where(decYears==years_state[i])[0] + + vol_snapshot[i] = vol[ind_snap] + vaf_snapshot[i] = vaf[ind_snap] + gia_snapshot[i] = gia[ind_snap] + fia_snapshot[i] = fia[ind_snap] + days_snapshot[i] = daysSinceStart[ind_snap] + + if decYears[ind_snap] == endYr: + break + + # this is for the flux variables + for i in range(nt_flux): + ind_avg = np.where(np.logical_and(decYears > years_flux[i], + decYears <= (years_flux[i] + 1.0)))[0] + smbi = smb[ind_avg] + bmbGri = bmbGr[ind_avg] + bmbFlti = bmbFlt[ind_avg] + cfxi = cfx[ind_avg] + fmfxi = fmfx[ind_avg] + gfxi = gfx[ind_avg] + dti = dt[ind_avg] + + # take the average of the flux variables + smb_avg[i] = np.nansum(smbi * dti) / np.nansum(dti) + bmbGr_avg[i] = np.nansum(bmbGri * dti) / np.nansum(dti) + bmbFlt_avg[i] = np.nansum(bmbFlti * dti) / np.nansum(dti) + cfx_avg[i] = np.nansum(cfxi * dti) / np.nansum(dti) + fmfx_avg[i] = np.nansum(fmfxi * dti) / np.nansum(dti) + gfx_avg[i] = np.nansum(gfxi * dti) / np.nansum(dti) + days_min[i] = (years_flux[i] - refYear) * 365.0 + days_max[i] = (years_flux[i] + 1.0 - refYear) * 365.0 + + if decYears[ind_avg][-1] == endYr: + break + + # -------------- lim ------------------ + data_scalar = Dataset(f'{output_path}/lim_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + limValues = data_scalar.createVariable('lim', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + limValues[i] = vol_snapshot[i] * 910 + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + limValues.standard_name = 'land_ice_mass' + limValues.units = 'kg' + data_scalar.AUTHORS = AUTHOR_STR + data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP= 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total ice mass' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- limnsw ------------------ + data_scalar = Dataset(f'{output_path}/limnsw_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + limnswValues = data_scalar.createVariable('limnsw', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + limnswValues[i] = vaf_snapshot[i] * 910 + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + limnswValues.standard_name = 'land_ice_mass_not_displacing_sea_water' + limnswValues.units = 'kg' + data_scalar.AUTHORS = AUTHOR_STR + data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Mass above floatation' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- iareagr ------------------ + data_scalar = Dataset(f'{output_path}/iareagr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + iareagrValues = data_scalar.createVariable('iareagr', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + iareagrValues[i] = gia_snapshot[i] + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + iareagrValues.standard_name = 'grounded_ice_sheet_area' + iareagrValues.units = 'm2' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Grounded ice area' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- iareafl ------------------ + data_scalar = Dataset(f'{output_path}/iareafl_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_state) + iareaflValues = data_scalar.createVariable('iareafl', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + for i in range(nt_state): + iareaflValues[i] = fia_snapshot[i] + timeValues[i] = days_snapshot[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + iareaflValues.standard_name = 'floating_ice_shelf_area' + iareaflValues.units = 'm2' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Floating ice area' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendacabf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendacabf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendacabfValues = data_scalar.createVariable('tendacabf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendacabfValues[i] = smb_avg[i] / 31536000.0 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendacabfValues.standard_name = 'tendency_of_land_ice_mass_due_to_surface_mass_balance' + tendacabfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total SMB flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlibmassbfgr: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlibmassbfgr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlibmassbfgrValues = data_scalar.createVariable('tendlibmassbfgr', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlibmassbfgrValues[i] = bmbGr_avg[i] / 31536000.0 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlibmassbfgrValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' + tendlibmassbfgrValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total BMB flux beneath grounded ice' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlibmassbffl: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlibmassbffl_AIS_DOE_MALI_{exp}.nc', 'w', + format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlibmassbfflValues = data_scalar.createVariable('tendlibmassbffl', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlibmassbfflValues[i] = bmbFlt_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlibmassbfflValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' + tendlibmassbfflValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total BMB flux beneath floating ice' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlicalvf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendlicalvf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlicalvfValues = data_scalar.createVariable('tendlicalvf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlicalvfValues[i] = -cfx_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlicalvfValues.standard_name = 'tendency_of_land_ice_mass_due_to_calving' + tendlicalvfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total calving flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendlifmassbf: this is a flux var + # In ISMIP6, this variable used to be 'Total calving and ice front melting flux' + # In ISMIP7, it represents 'Total ice front melting flux' only, without calving flux + data_scalar = Dataset(f'{output_path}/tendlifmassbf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendlifmassbfValues = data_scalar.createVariable('tendlifmassbf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendlifmassbfValues[i] = -fmfx_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendlifmassbfValues.standard_name = 'tendency_of_land_ice_mass_due_to_ice_front_melting' + tendlifmassbfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total ice front melting flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + # -------------- tendligroundf: this is a flux var + data_scalar = Dataset(f'{output_path}/tendligroundf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') + data_scalar.createDimension('time', nt_flux) + tendligroundfValues = data_scalar.createVariable('tendligroundf', 'd', ('time')) + timeValues = data_scalar.createVariable('time', 'd', ('time')) + data_scalar.createDimension('bnds', 2) + timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) + for i in range(nt_flux): + tendligroundfValues[i] = gfx_avg[i] / 31536000 + timeValues[i] = (days_min[i] + days_max[i]) / 2.0 + timebndsValues[i, 0] = days_min[i] + timebndsValues[i, 1] = days_max[i] + timeValues.units = f'days since {simulationStartDate}' + timeValues.calendar = 'noleap' + timeValues.standard_name = 'time' + timeValues.long_name = 'time' + tendligroundfValues.standard_name = 'tendency_of_grounded_ice_mass' + tendligroundfValues.units = 'kg s-1' + data_scalar.AUTHORS= AUTHOR_STR + data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' + data_scalar.GROUP = 'Los Alamos National Laboratory' + data_scalar.VARIABLE = 'Total grounding line flux' + data_scalar.DATE = DATE_STR + data_scalar.close() + + data.close() diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py old mode 100755 new mode 100644 index f95c2f4bc..bbdd66bec --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -283,395 +283,3 @@ def generate_output_2d_state_vars(file_remapped_mali_state, 'Floating ice shelf area fraction', file_remapped_mali_state, ismip7_grid_file, exp, output_path) - - -def generate_output_1d_vars(global_stats_file, exp, output_path=None): - """ - This code processes both state and flux 1D variables - global_stats_file: MALI globalStats.nc output file - exp: ISMIP7 experiment number - output_path: - """ - - if not os.path.exists(output_path): - output_path = os.getcwd() - - AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' - DATE_STR = date.today().strftime("%d-%b-%Y") - - data = Dataset(global_stats_file, 'r') - nt_in = len(data.dimensions['Time']) - xtime = data.variables['xtime'][:, :] - daysSinceStart = data.variables['daysSinceStart'][:] - dt = data.variables['deltat'][:] - simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') - simulationStartDate = simulationStartTime.split("_")[0] - if simulationStartDate[5:10] != '01-01': - sys.exit("Error: simulationStartTime for globalStats file is not on Jan. 1.") - refYear = int(simulationStartDate[0:4]) - decYears = refYear + daysSinceStart/365.0 - endYr = decYears[-1] - if endYr != np.round(endYr): - sys.exit("Error: end year not an even year in globalStats file.") - - # Determine processed time levels for state and flux fields - # The historical state fields should include the initial time (Jan. 1). - # Projection state fields should not include the initial time (Jan. 1) - # of the projection because it's a restart from the historical. - # Flux fields should never use the Jan. 1 time level at the start of the - # year as part of the averaging. - # For year conventions here, for state fields, the year is the snapshot at - # the start of the year, e.g., state year 2000 means the snapshot at Jan. 1, 2000. - # For flux fields, the years is the calendar year being averaged over, - # e.g., flux year 2000 is the average between Jan. 1, 2000, and Jan. 1, 2001. - # Note this year convention differs from the first column in table in A2.3.2 at - # https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP7-Projections2300-Antarctica#A2.3.3_Table_A1:_Variable_request_for_ISMIP6 - # but that year indexing convention ultimately doesn't matter because the - # time coordinates in these files uses units of days since a reference date, - # and it does not use a year indexing convention at all. - if decYears[0] == np.round(decYears[0]): - # The initial time level will only be on an even year (Jan. 1) - # for the hist run. In that case, we want to include that initial - # even year in the state processing. We also want the state snapshot - # at the final (even) year in the output. - # The flux processing should start with the first year, which covers a - # full 12 months. We exclude the final year, which is just a Jan. 1 posting. - years_state = np.arange(decYears[0], endYr + 1) - years_flux = np.arange(decYears[0], endYr) - else: - # For projection runs, the first state snapshot we want is the first Jan. 1, - # which we be the first even year after the initial time in the file. - # For flux files, the first full year we want to process is the year of the - # first time level in the file. As with hist, we exclude the final year, - # which is just a Jan. 1 posting. - years_state = np.arange(np.ceil(decYears[0]), endYr + 1) - years_flux = np.arange(np.floor(decYears[0]), endYr) - nt_state = len(years_state) - nt_flux = len(years_flux) - print(f'For state processing, using start year={years_state[0]} and end year={years_state[-1]}.') - print(f'For flux processing, using start year={years_flux[0]} and end year={years_flux[-1]}.') - - # read in state variables - vol = data.variables['totalIceVolume'][:] - vaf = data.variables['volumeAboveFloatation'][:] - gia = data.variables['groundedIceArea'][:] - fia = data.variables['floatingIceArea'][:] - - # read in flux variables over which yearly average will be taken - smb = data.variables['totalSfcMassBal'][:] - bmbGr = data.variables['totalGroundedBasalMassBal'][:] - # clean out some garbage values we can't account for - ind = np.nonzero(bmbGr > 1.0e18)[0] - if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal>1.0e18") - bmbGr[ind] = np.nan - ind = np.nonzero(bmbGr < -1.0e18)[0] - if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal<-1.0e18") - bmbGr[ind] = np.nan - bmbFlt = data.variables['totalFloatingBasalMassBal'][:] - # clean out some garbage values we can't account for - ind = np.nonzero(bmbFlt>1.0e18)[0] - if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal>1.0e18") - bmbFlt[ind] = np.nan - ind = np.nonzero(bmbFlt<-1.0e18)[0] - if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal<-1.0e18") - bmbFlt[ind] = np.nan - cfx = data.variables['totalCalvingFlux'][:] - fmfx = data.variables['totalFaceMeltingFlux'][:] - gfx = data.variables['groundingLineFlux'][:] - - # initialize 1D variables that will store data value on the - # January 1st of each year - vol_snapshot = np.zeros(nt_state) * np.nan - vaf_snapshot = np.zeros(nt_state) * np.nan - gia_snapshot = np.zeros(nt_state) * np.nan - fia_snapshot = np.zeros(nt_state) * np.nan - days_snapshot = np.zeros(nt_state) * np.nan - smb_avg = np.zeros(nt_flux) * np.nan - bmbGr_avg = np.zeros(nt_flux) * np.nan - bmbFlt_avg = np.zeros(nt_flux) * np.nan - cfx_avg = np.zeros(nt_flux) * np.nan - fmfx_avg = np.zeros(nt_flux) * np.nan - gfx_avg = np.zeros(nt_flux) * np.nan - days_min = np.zeros(nt_flux) * np.nan - days_max = np.zeros(nt_flux) * np.nan - - # this is for the state variables - for i in range(nt_state): - ind_snap = np.where(decYears==years_state[i])[0] - - vol_snapshot[i] = vol[ind_snap] - vaf_snapshot[i] = vaf[ind_snap] - gia_snapshot[i] = gia[ind_snap] - fia_snapshot[i] = fia[ind_snap] - days_snapshot[i] = daysSinceStart[ind_snap] - - if decYears[ind_snap] == endYr: - break - - # this is for the flux variables - for i in range(nt_flux): - ind_avg = np.where(np.logical_and(decYears > years_flux[i], - decYears <= (years_flux[i] + 1.0)))[0] - smbi = smb[ind_avg] - bmbGri = bmbGr[ind_avg] - bmbFlti = bmbFlt[ind_avg] - cfxi = cfx[ind_avg] - fmfxi = fmfx[ind_avg] - gfxi = gfx[ind_avg] - dti = dt[ind_avg] - - # take the average of the flux variables - smb_avg[i] = np.nansum(smbi * dti) / np.nansum(dti) - bmbGr_avg[i] = np.nansum(bmbGri * dti) / np.nansum(dti) - bmbFlt_avg[i] = np.nansum(bmbFlti * dti) / np.nansum(dti) - cfx_avg[i] = np.nansum(cfxi * dti) / np.nansum(dti) - fmfx_avg[i] = np.nansum(fmfxi * dti) / np.nansum(dti) - gfx_avg[i] = np.nansum(gfxi * dti) / np.nansum(dti) - days_min[i] = (years_flux[i] - refYear) * 365.0 - days_max[i] = (years_flux[i] + 1.0 - refYear) * 365.0 - - if decYears[ind_avg][-1] == endYr: - break - - # -------------- lim ------------------ - data_scalar = Dataset(f'{output_path}/lim_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - limValues = data_scalar.createVariable('lim', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - limValues[i] = vol_snapshot[i] * 910 - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - limValues.standard_name = 'land_ice_mass' - limValues.units = 'kg' - data_scalar.AUTHORS = AUTHOR_STR - data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP= 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total ice mass' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- limnsw ------------------ - data_scalar = Dataset(f'{output_path}/limnsw_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - limnswValues = data_scalar.createVariable('limnsw', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - limnswValues[i] = vaf_snapshot[i] * 910 - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - limnswValues.standard_name = 'land_ice_mass_not_displacing_sea_water' - limnswValues.units = 'kg' - data_scalar.AUTHORS = AUTHOR_STR - data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Mass above floatation' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- iareagr ------------------ - data_scalar = Dataset(f'{output_path}/iareagr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - iareagrValues = data_scalar.createVariable('iareagr', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - iareagrValues[i] = gia_snapshot[i] - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - iareagrValues.standard_name = 'grounded_ice_sheet_area' - iareagrValues.units = 'm2' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Grounded ice area' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- iareafl ------------------ - data_scalar = Dataset(f'{output_path}/iareafl_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - iareaflValues = data_scalar.createVariable('iareafl', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - iareaflValues[i] = fia_snapshot[i] - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - iareaflValues.standard_name = 'floating_ice_shelf_area' - iareaflValues.units = 'm2' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Floating ice area' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendacabf: this is a flux var - data_scalar = Dataset(f'{output_path}/tendacabf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendacabfValues = data_scalar.createVariable('tendacabf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendacabfValues[i] = smb_avg[i] / 31536000.0 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendacabfValues.standard_name = 'tendency_of_land_ice_mass_due_to_surface_mass_balance' - tendacabfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total SMB flux' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlibmassbfgr: this is a flux var - data_scalar = Dataset(f'{output_path}/tendlibmassbfgr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlibmassbfgrValues = data_scalar.createVariable('tendlibmassbfgr', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlibmassbfgrValues[i] = bmbGr_avg[i] / 31536000.0 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlibmassbfgrValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' - tendlibmassbfgrValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total BMB flux beneath grounded ice' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlibmassbffl: this is a flux var - data_scalar = Dataset(f'{output_path}/tendlibmassbffl_AIS_DOE_MALI_{exp}.nc', 'w', - format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlibmassbfflValues = data_scalar.createVariable('tendlibmassbffl', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlibmassbfflValues[i] = bmbFlt_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlibmassbfflValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' - tendlibmassbfflValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total BMB flux beneath floating ice' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlicalvf: this is a flux var - data_scalar = Dataset(f'{output_path}/tendlicalvf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlicalvfValues = data_scalar.createVariable('tendlicalvf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlicalvfValues[i] = -cfx_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlicalvfValues.standard_name = 'tendency_of_land_ice_mass_due_to_calving' - tendlicalvfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total calving flux' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlifmassbf: this is a flux var - # In ISMIP6, this variable used to be 'Total calving and ice front melting flux' - # In ISMIP7, it represents 'Total ice front melting flux' only, without calving flux - data_scalar = Dataset(f'{output_path}/tendlifmassbf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlifmassbfValues = data_scalar.createVariable('tendlifmassbf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlifmassbfValues[i] = -fmfx_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlifmassbfValues.standard_name = 'tendency_of_land_ice_mass_due_to_ice_front_melting' - tendlifmassbfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total ice front melting flux' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendligroundf: this is a flux var - data_scalar = Dataset(f'{output_path}/tendligroundf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendligroundfValues = data_scalar.createVariable('tendligroundf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendligroundfValues[i] = gfx_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendligroundfValues.standard_name = 'tendency_of_grounded_ice_mass' - tendligroundfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total grounding line flux' - data_scalar.DATE = DATE_STR - data_scalar.close() - - data.close() From 87d861a0f619f2a932399e94576f7c4ac0a90259 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 09:57:29 -0700 Subject: [PATCH 12/33] Convert 1d processing to xarray This eliminates need to cat files ahead of time. --- .../post_process_mali_to_ismip7.py | 23 +-- .../process_1d_variables_ismip7.py | 142 +++++++++++++++--- 2 files changed, 132 insertions(+), 33 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 049af6e4d..d84578b82 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -11,12 +11,14 @@ import argparse from subprocess import check_call import os +import glob from datetime import datetime from grid_and_mapping import build_mapping_file, check_ismip7_grid_file, \ check_exp_name from process_state_variables_ismip7 import generate_output_2d_state_vars, \ process_state_vars -from process_1d_variables_ismip7 import generate_output_1d_vars +from process_1d_variables_ismip7 import generate_output_1d_vars, \ + check_global_stats_files from process_flux_variables_ismip7 import generate_output_2d_flux_vars def main(): @@ -32,8 +34,10 @@ def main(): required=False, help="mpas output flux variables") parser.add_argument("-i_mesh", "--input_mesh", dest="input_file_grid", required=False, help="MALI file with mesh information") - parser.add_argument("-g", "--global_stats_file", dest="global_stats_file", - required=False, help="globalStats.nc file") + parser.add_argument("-g", "--global_stats_pattern", dest="global_stats_pattern", + required=False, + help="glob pattern matching one or more globalStats.nc " +onver "files (e.g. 'globalStats_*.nc')") parser.add_argument("-p", "--output_path", dest="output_path", required=False, help="path to which the final output files" @@ -94,13 +98,14 @@ def main(): os.makedirs(output_path) # process 1D variables - if args.global_stats_file is None: - print("--- MALI global stats file is not provided, thus it will not be processed.") + if args.global_stats_pattern is None: + print("--- No global stats pattern provided; skipping 1D variable processing.") else: - print("\n---Processing global stats file---") - generate_output_1d_vars(args.global_stats_file, args.exp, - output_path) - print("---Processing global stats file complete---\n") + print("\n---Processing global stats file(s)---") + global_stats_files = sorted(glob.glob(args.global_stats_pattern)) + check_global_stats_files(global_stats_files) + generate_output_1d_vars(global_stats_files, args.exp, output_path) + print("---Processing global stats file(s) complete---\n") # process 2d state variables if args.input_file_state is None: diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index f0f1fa75a..4c5c7f852 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -8,34 +8,128 @@ import numpy as np import os import sys +import glob +import xarray as xr -def generate_output_1d_vars(global_stats_file, exp, output_path=None): + +EXPECTED_VARIABLES = [ + 'daysSinceStart', 'deltat', 'simulationStartTime', + 'totalIceVolume', 'volumeAboveFloatation', + 'groundedIceArea', 'floatingIceArea', + 'totalSfcMassBal', 'totalGroundedBasalMassBal', + 'totalFloatingBasalMassBal', 'totalCalvingFlux', + 'totalFaceMeltingFlux', 'groundingLineFlux', +] + + +def check_global_stats_files(files): + """ + Validate a list of globalStats files before processing. + + Checks that: + - The list is not empty + - Each file exists + - Each file contains the expected variables + - simulationStartTime is consistent across all files + - No time overlaps exist between consecutive files + - No unexpectedly large time gaps (> 366 days) exist between consecutive files + + Parameters + ---------- + files : list of str + Sorted list of globalStats file paths. + + Raises + ------ + ValueError + If any validation check fails. + FileNotFoundError + If any file does not exist. + """ + if len(files) == 0: + raise ValueError( + "No globalStats files matched the provided glob pattern.") + + for f in files: + if not os.path.exists(f): + raise FileNotFoundError(f"globalStats file not found: {f}") + + # Check required variables in each file + for f in files: + with xr.open_dataset(f, decode_cf=False) as ds: + missing = [v for v in EXPECTED_VARIABLES if v not in ds] + if missing: + raise ValueError( + f"File '{f}' is missing expected variables: {missing}") + + # Check simulationStartTime consistency across all files + start_times = [] + for f in files: + with xr.open_dataset(f, decode_cf=False) as ds: + start_times.append( + ds['simulationStartTime'].values.tobytes() + .decode('utf-8').strip().strip('\x00')) + if len(set(start_times)) > 1: + raise ValueError( + f"Inconsistent simulationStartTime across globalStats files: " + f"{set(start_times)}") + + # Check for time overlaps or large gaps between consecutive files + for i in range(len(files) - 1): + with xr.open_dataset(files[i], decode_cf=False) as ds_a: + end_a = float(ds_a['daysSinceStart'].values[-1]) + with xr.open_dataset(files[i + 1], decode_cf=False) as ds_b: + start_b = float(ds_b['daysSinceStart'].values[0]) + if start_b <= end_a: + raise ValueError( + f"Time overlap detected between files:\n" + f" {files[i]} (ends at day {end_a})\n" + f" {files[i + 1]} (starts at day {start_b})") + gap_days = start_b - end_a + if gap_days > 366: + print(f"WARNING: Gap of {gap_days:.1f} days between files:\n" + f" {files[i]}\n {files[i + 1]}") + + print(f"Validated {len(files)} globalStats file(s).") + + +def generate_output_1d_vars(files, exp, output_path=None): """ - This code processes both state and flux 1D variables - global_stats_file: MALI globalStats.nc output file - exp: ISMIP7 experiment number - output_path: + Process and write 1D (scalar time-series) state and flux variables. + + Parameters + ---------- + files : list of str + Sorted list of globalStats.nc file paths to process. + exp : str + ISMIP7 experiment name (e.g. 'C001'). + output_path : str, optional + Directory for output files. Defaults to current working directory. """ - if not os.path.exists(output_path): + if output_path is None or not os.path.exists(output_path): output_path = os.getcwd() AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' DATE_STR = date.today().strftime("%d-%b-%Y") - data = Dataset(global_stats_file, 'r') - nt_in = len(data.dimensions['Time']) - xtime = data.variables['xtime'][:, :] - daysSinceStart = data.variables['daysSinceStart'][:] - dt = data.variables['deltat'][:] - simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + ds = xr.open_mfdataset(files, combine='nested', concat_dim='Time', + decode_cf=False, data_vars='minimal', + coords='minimal', compat='override') + with xr.open_dataset(files[0], decode_cf=False) as ds_first: + simulationStartTime = (ds_first['simulationStartTime'].values + .tobytes().decode('utf-8').strip().strip('\x00')) + daysSinceStart = ds['daysSinceStart'].values + dt = ds['deltat'].values simulationStartDate = simulationStartTime.split("_")[0] if simulationStartDate[5:10] != '01-01': + ds.close() sys.exit("Error: simulationStartTime for globalStats file is not on Jan. 1.") refYear = int(simulationStartDate[0:4]) - decYears = refYear + daysSinceStart/365.0 + decYears = refYear + daysSinceStart / 365.0 endYr = decYears[-1] if endYr != np.round(endYr): + ds.close() sys.exit("Error: end year not an even year in globalStats file.") # Determine processed time levels for state and flux fields @@ -76,14 +170,14 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): print(f'For flux processing, using start year={years_flux[0]} and end year={years_flux[-1]}.') # read in state variables - vol = data.variables['totalIceVolume'][:] - vaf = data.variables['volumeAboveFloatation'][:] - gia = data.variables['groundedIceArea'][:] - fia = data.variables['floatingIceArea'][:] + vol = ds['totalIceVolume'].values + vaf = ds['volumeAboveFloatation'].values + gia = ds['groundedIceArea'].values + fia = ds['floatingIceArea'].values # read in flux variables over which yearly average will be taken - smb = data.variables['totalSfcMassBal'][:] - bmbGr = data.variables['totalGroundedBasalMassBal'][:] + smb = ds['totalSfcMassBal'].values + bmbGr = ds['totalGroundedBasalMassBal'].values.copy() # clean out some garbage values we can't account for ind = np.nonzero(bmbGr > 1.0e18)[0] if len(ind) > 0: @@ -93,7 +187,7 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): if len(ind) > 0: print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal<-1.0e18") bmbGr[ind] = np.nan - bmbFlt = data.variables['totalFloatingBasalMassBal'][:] + bmbFlt = ds['totalFloatingBasalMassBal'].values.copy() # clean out some garbage values we can't account for ind = np.nonzero(bmbFlt>1.0e18)[0] if len(ind) > 0: @@ -103,9 +197,9 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): if len(ind) > 0: print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal<-1.0e18") bmbFlt[ind] = np.nan - cfx = data.variables['totalCalvingFlux'][:] - fmfx = data.variables['totalFaceMeltingFlux'][:] - gfx = data.variables['groundingLineFlux'][:] + cfx = ds['totalCalvingFlux'].values + fmfx = ds['totalFaceMeltingFlux'].values + gfx = ds['groundingLineFlux'].values # initialize 1D variables that will store data value on the # January 1st of each year @@ -398,4 +492,4 @@ def generate_output_1d_vars(global_stats_file, exp, output_path=None): data_scalar.DATE = DATE_STR data_scalar.close() - data.close() + ds.close() From 731553c28e1561cb5fc97ebff27fa4d1a345fe02 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 10:01:55 -0700 Subject: [PATCH 13/33] add fns for writing 1d vars to reduce code reuse --- .../process_1d_variables_ismip7.py | 333 +++++------------- 1 file changed, 97 insertions(+), 236 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index 4c5c7f852..fad16ca43 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -93,6 +93,62 @@ def check_global_stats_files(files): print(f"Validated {len(files)} globalStats file(s).") +def _write_state_var(varname, data_values, time_days, standard_name, units, + variable_desc, output_path, simulationStartDate, + author_str, date_str, exp): + """Write a single 1D state (snapshot) variable to a NETCDF4_CLASSIC file.""" + nt = len(data_values) + ds_out = Dataset(f'{output_path}/{varname}_AIS_DOE_MALI_{exp}.nc', + 'w', format='NETCDF4_CLASSIC') + ds_out.createDimension('time', nt) + var_out = ds_out.createVariable(varname, 'd', ('time',)) + time_out = ds_out.createVariable('time', 'd', ('time',)) + var_out[:] = data_values + time_out[:] = time_days + time_out.units = f'days since {simulationStartDate}' + time_out.calendar = 'noleap' + time_out.standard_name = 'time' + time_out.long_name = 'time' + var_out.standard_name = standard_name + var_out.units = units + ds_out.AUTHORS = author_str + ds_out.MODEL = 'MALI (MPAS-Albany Land Ice model)' + ds_out.GROUP = 'Los Alamos National Laboratory' + ds_out.VARIABLE = variable_desc + ds_out.DATE = date_str + ds_out.close() + + +def _write_flux_var(varname, data_values, days_min, days_max, standard_name, + units, variable_desc, output_path, simulationStartDate, + author_str, date_str, exp): + """Write a single 1D flux (time-averaged) variable to a NETCDF4_CLASSIC file.""" + nt = len(data_values) + ds_out = Dataset(f'{output_path}/{varname}_AIS_DOE_MALI_{exp}.nc', + 'w', format='NETCDF4_CLASSIC') + ds_out.createDimension('time', nt) + ds_out.createDimension('bnds', 2) + var_out = ds_out.createVariable(varname, 'd', ('time',)) + time_out = ds_out.createVariable('time', 'd', ('time',)) + bnds_out = ds_out.createVariable('time_bnds', 'd', ('time', 'bnds')) + var_out[:] = data_values + time_out[:] = (days_min + days_max) / 2.0 + bnds_out[:, 0] = days_min + bnds_out[:, 1] = days_max + time_out.units = f'days since {simulationStartDate}' + time_out.calendar = 'noleap' + time_out.standard_name = 'time' + time_out.long_name = 'time' + var_out.standard_name = standard_name + var_out.units = units + ds_out.AUTHORS = author_str + ds_out.MODEL = 'MALI (MPAS-Albany Land Ice model)' + ds_out.GROUP = 'Los Alamos National Laboratory' + ds_out.VARIABLE = variable_desc + ds_out.DATE = date_str + ds_out.close() + + def generate_output_1d_vars(files, exp, output_path=None): """ Process and write 1D (scalar time-series) state and flux variables. @@ -255,241 +311,46 @@ def generate_output_1d_vars(files, exp, output_path=None): if decYears[ind_avg][-1] == endYr: break - # -------------- lim ------------------ - data_scalar = Dataset(f'{output_path}/lim_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - limValues = data_scalar.createVariable('lim', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - limValues[i] = vol_snapshot[i] * 910 - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - limValues.standard_name = 'land_ice_mass' - limValues.units = 'kg' - data_scalar.AUTHORS = AUTHOR_STR - data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP= 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total ice mass' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- limnsw ------------------ - data_scalar = Dataset(f'{output_path}/limnsw_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - limnswValues = data_scalar.createVariable('limnsw', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - limnswValues[i] = vaf_snapshot[i] * 910 - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - limnswValues.standard_name = 'land_ice_mass_not_displacing_sea_water' - limnswValues.units = 'kg' - data_scalar.AUTHORS = AUTHOR_STR - data_scalar.MODEL = 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Mass above floatation' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- iareagr ------------------ - data_scalar = Dataset(f'{output_path}/iareagr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - iareagrValues = data_scalar.createVariable('iareagr', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - iareagrValues[i] = gia_snapshot[i] - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - iareagrValues.standard_name = 'grounded_ice_sheet_area' - iareagrValues.units = 'm2' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Grounded ice area' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- iareafl ------------------ - data_scalar = Dataset(f'{output_path}/iareafl_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_state) - iareaflValues = data_scalar.createVariable('iareafl', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - for i in range(nt_state): - iareaflValues[i] = fia_snapshot[i] - timeValues[i] = days_snapshot[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - iareaflValues.standard_name = 'floating_ice_shelf_area' - iareaflValues.units = 'm2' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Floating ice area' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendacabf: this is a flux var - data_scalar = Dataset(f'{output_path}/tendacabf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendacabfValues = data_scalar.createVariable('tendacabf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendacabfValues[i] = smb_avg[i] / 31536000.0 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendacabfValues.standard_name = 'tendency_of_land_ice_mass_due_to_surface_mass_balance' - tendacabfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total SMB flux' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlibmassbfgr: this is a flux var - data_scalar = Dataset(f'{output_path}/tendlibmassbfgr_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlibmassbfgrValues = data_scalar.createVariable('tendlibmassbfgr', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlibmassbfgrValues[i] = bmbGr_avg[i] / 31536000.0 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlibmassbfgrValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' - tendlibmassbfgrValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total BMB flux beneath grounded ice' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlibmassbffl: this is a flux var - data_scalar = Dataset(f'{output_path}/tendlibmassbffl_AIS_DOE_MALI_{exp}.nc', 'w', - format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlibmassbfflValues = data_scalar.createVariable('tendlibmassbffl', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlibmassbfflValues[i] = bmbFlt_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlibmassbfflValues.standard_name = 'tendency_of_land_ice_mass_due_to_basal_mass_balance' - tendlibmassbfflValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total BMB flux beneath floating ice' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlicalvf: this is a flux var - data_scalar = Dataset(f'{output_path}/tendlicalvf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlicalvfValues = data_scalar.createVariable('tendlicalvf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlicalvfValues[i] = -cfx_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlicalvfValues.standard_name = 'tendency_of_land_ice_mass_due_to_calving' - tendlicalvfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total calving flux' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendlifmassbf: this is a flux var - # In ISMIP6, this variable used to be 'Total calving and ice front melting flux' - # In ISMIP7, it represents 'Total ice front melting flux' only, without calving flux - data_scalar = Dataset(f'{output_path}/tendlifmassbf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendlifmassbfValues = data_scalar.createVariable('tendlifmassbf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendlifmassbfValues[i] = -fmfx_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendlifmassbfValues.standard_name = 'tendency_of_land_ice_mass_due_to_ice_front_melting' - tendlifmassbfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total ice front melting flux' - data_scalar.DATE = DATE_STR - data_scalar.close() - - # -------------- tendligroundf: this is a flux var - data_scalar = Dataset(f'{output_path}/tendligroundf_AIS_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') - data_scalar.createDimension('time', nt_flux) - tendligroundfValues = data_scalar.createVariable('tendligroundf', 'd', ('time')) - timeValues = data_scalar.createVariable('time', 'd', ('time')) - data_scalar.createDimension('bnds', 2) - timebndsValues = data_scalar.createVariable('time_bnds', 'd', ('time', 'bnds')) - for i in range(nt_flux): - tendligroundfValues[i] = gfx_avg[i] / 31536000 - timeValues[i] = (days_min[i] + days_max[i]) / 2.0 - timebndsValues[i, 0] = days_min[i] - timebndsValues[i, 1] = days_max[i] - timeValues.units = f'days since {simulationStartDate}' - timeValues.calendar = 'noleap' - timeValues.standard_name = 'time' - timeValues.long_name = 'time' - tendligroundfValues.standard_name = 'tendency_of_grounded_ice_mass' - tendligroundfValues.units = 'kg s-1' - data_scalar.AUTHORS= AUTHOR_STR - data_scalar.MODEL= 'MALI (MPAS-Albany Land Ice model)' - data_scalar.GROUP = 'Los Alamos National Laboratory' - data_scalar.VARIABLE = 'Total grounding line flux' - data_scalar.DATE = DATE_STR - data_scalar.close() + # shared keyword arguments for both writer helpers + common = dict( + output_path=output_path, + simulationStartDate=simulationStartDate, + author_str=AUTHOR_STR, + date_str=DATE_STR, + exp=exp, + ) + + # --- state (snapshot) variables --- + _write_state_var('lim', vol_snapshot * 910, days_snapshot, + 'land_ice_mass', 'kg', 'Total ice mass', **common) + _write_state_var('limnsw', vaf_snapshot * 910, days_snapshot, + 'land_ice_mass_not_displacing_sea_water', 'kg', + 'Mass above floatation', **common) + _write_state_var('iareagr', gia_snapshot, days_snapshot, + 'grounded_ice_sheet_area', 'm2', 'Grounded ice area', **common) + _write_state_var('iareafl', fia_snapshot, days_snapshot, + 'floating_ice_shelf_area', 'm2', 'Floating ice area', **common) + + # --- flux (time-averaged) variables --- + _write_flux_var('tendacabf', smb_avg / 31536000.0, days_min, days_max, + 'tendency_of_land_ice_mass_due_to_surface_mass_balance', + 'kg s-1', 'Total SMB flux', **common) + _write_flux_var('tendlibmassbfgr', bmbGr_avg / 31536000.0, days_min, days_max, + 'tendency_of_land_ice_mass_due_to_basal_mass_balance', + 'kg s-1', 'Total BMB flux beneath grounded ice', **common) + _write_flux_var('tendlibmassbffl', bmbFlt_avg / 31536000.0, days_min, days_max, + 'tendency_of_land_ice_mass_due_to_basal_mass_balance', + 'kg s-1', 'Total BMB flux beneath floating ice', **common) + # tendlicalvf: sign convention — calving removes mass, so negate + _write_flux_var('tendlicalvf', -cfx_avg / 31536000.0, days_min, days_max, + 'tendency_of_land_ice_mass_due_to_calving', + 'kg s-1', 'Total calving flux', **common) + # tendlifmassbf: in ISMIP7 this is ice-front melting only (not calving) + _write_flux_var('tendlifmassbf', -fmfx_avg / 31536000.0, days_min, days_max, + 'tendency_of_land_ice_mass_due_to_ice_front_melting', + 'kg s-1', 'Total ice front melting flux', **common) + _write_flux_var('tendligroundf', gfx_avg / 31536000.0, days_min, days_max, + 'tendency_of_grounded_ice_mass', + 'kg s-1', 'Total grounding line flux', **common) ds.close() From f425fe6a0822b996dc23a09ac6734f4b9b4e6761 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 10:08:13 -0700 Subject: [PATCH 14/33] Add icesheet arg to support both AIS and GIS --- .../post_process_mali_to_ismip7.py | 13 +++++++++---- .../process_1d_variables_ismip7.py | 13 ++++++++----- .../process_flux_variables_ismip7.py | 5 +++-- .../process_state_variables_ismip7.py | 5 +++-- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index d84578b82..e1a5a2dc2 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -37,7 +37,7 @@ def main(): parser.add_argument("-g", "--global_stats_pattern", dest="global_stats_pattern", required=False, help="glob pattern matching one or more globalStats.nc " -onver "files (e.g. 'globalStats_*.nc')") + "files (e.g. 'globalStats_*.nc')") parser.add_argument("-p", "--output_path", dest="output_path", required=False, help="path to which the final output files" @@ -53,6 +53,10 @@ def main(): parser.add_argument("--res", dest="res_ismip7_grid", required=True, help="resolution of the ismip7 grid, (e.g. 8 for 8km res)") + parser.add_argument("--icesheet", dest="icesheet", + required=True, + choices=['AIS', 'GIS'], + help="ice sheet domain: 'AIS' (Antarctica) or 'GIS' (Greenland)") args = parser.parse_args() check_exp_name(args.exp) @@ -104,7 +108,8 @@ def main(): print("\n---Processing global stats file(s)---") global_stats_files = sorted(glob.glob(args.global_stats_pattern)) check_global_stats_files(global_stats_files) - generate_output_1d_vars(global_stats_files, args.exp, output_path) + generate_output_1d_vars(global_stats_files, args.exp, args.icesheet, + output_path) print("---Processing global stats file(s) complete---\n") # process 2d state variables @@ -134,7 +139,7 @@ def main(): print("Writing processed and remapped state fields to ISMIP7 file format.") generate_output_2d_state_vars(processed_and_remapped_file_state, args.ismip7_grid_file, - args.exp, output_path) + args.exp, output_path, args.icesheet) os.remove(tmp_file) os.remove(processed_and_remapped_file_state) @@ -159,7 +164,7 @@ def main(): # write out the output files in the ismip7-required format generate_output_2d_flux_vars(processed_file_flux, args.ismip7_grid_file, - args.exp, output_path) + args.exp, output_path, args.icesheet) cleanUp = True if cleanUp: diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index fad16ca43..3d22c9a6f 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -95,10 +95,10 @@ def check_global_stats_files(files): def _write_state_var(varname, data_values, time_days, standard_name, units, variable_desc, output_path, simulationStartDate, - author_str, date_str, exp): + author_str, date_str, exp, icesheet): """Write a single 1D state (snapshot) variable to a NETCDF4_CLASSIC file.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_AIS_DOE_MALI_{exp}.nc', + ds_out = Dataset(f'{output_path}/{varname}_{icesheet}_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) var_out = ds_out.createVariable(varname, 'd', ('time',)) @@ -121,10 +121,10 @@ def _write_state_var(varname, data_values, time_days, standard_name, units, def _write_flux_var(varname, data_values, days_min, days_max, standard_name, units, variable_desc, output_path, simulationStartDate, - author_str, date_str, exp): + author_str, date_str, exp, icesheet): """Write a single 1D flux (time-averaged) variable to a NETCDF4_CLASSIC file.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_AIS_DOE_MALI_{exp}.nc', + ds_out = Dataset(f'{output_path}/{varname}_{icesheet}_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) ds_out.createDimension('bnds', 2) @@ -149,7 +149,7 @@ def _write_flux_var(varname, data_values, days_min, days_max, standard_name, ds_out.close() -def generate_output_1d_vars(files, exp, output_path=None): +def generate_output_1d_vars(files, exp, icesheet, output_path=None): """ Process and write 1D (scalar time-series) state and flux variables. @@ -159,6 +159,8 @@ def generate_output_1d_vars(files, exp, output_path=None): Sorted list of globalStats.nc file paths to process. exp : str ISMIP7 experiment name (e.g. 'C001'). + icesheet : str + Ice sheet domain, either 'AIS' or 'GIS'. output_path : str, optional Directory for output files. Defaults to current working directory. """ @@ -318,6 +320,7 @@ def generate_output_1d_vars(files, exp, output_path=None): author_str=AUTHOR_STR, date_str=DATE_STR, exp=exp, + icesheet=icesheet, ) # --- state (snapshot) variables --- diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index e327c3bac..153d1b54a 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -47,7 +47,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_AIS_DOE_MALI_{exp}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{icesheet}_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('bnds', 2) @@ -103,7 +103,8 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, def generate_output_2d_flux_vars(file_remapped_mali_flux, - ismip7_grid_file, exp, output_path): + ismip7_grid_file, exp, output_path, + icesheet): """ file_remapped_mali_flux: flux output file on mali mesh remapped onto the ismip7 grid diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index bbdd66bec..0e84ab122 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -89,7 +89,7 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_AIS_DOE_MALI_{exp}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{icesheet}_DOE_MALI_{exp}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('x', lonN) @@ -146,7 +146,8 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, def generate_output_2d_state_vars(file_remapped_mali_state, - ismip7_grid_file, exp, output_path): + ismip7_grid_file, exp, output_path, + icesheet): """ file_remapped_mali_state: output files on mali mesh remapped on the ismip7 grid From d8812cc9c24cc436936aed8fdab5016f86d64214 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 10:10:22 -0700 Subject: [PATCH 15/33] make output_path required --- .../ismip7_postprocessing/post_process_mali_to_ismip7.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index e1a5a2dc2..2c450b47f 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -39,7 +39,7 @@ def main(): help="glob pattern matching one or more globalStats.nc " "files (e.g. 'globalStats_*.nc')") parser.add_argument("-p", "--output_path", dest="output_path", - required=False, + required=True, help="path to which the final output files" " will be saved") parser.add_argument("--reuse_mapping_file", dest="reuse_mapping_file", @@ -93,10 +93,7 @@ def main(): print("---Processing remapping file complete---\n") # define the path to which the output (processed) files will be saved - if args.output_path is None: - output_path = os.getcwd() - else: - output_path = args.output_path + output_path = args.output_path print(f"Using output path: {output_path}") if not os.path.isdir(output_path): os.makedirs(output_path) From a9127701953aa9c3b550230b1902a2b2ebbee4e1 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 10:18:55 -0700 Subject: [PATCH 16/33] update docs and validate resolution, tweak CLAs --- .../ismip7_postprocessing/grid_and_mapping.py | 28 +++++++++++ .../post_process_mali_to_ismip7.py | 48 ++++++++++++------- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py index f89950185..04369a580 100644 --- a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py +++ b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py @@ -6,6 +6,34 @@ VALID_EXPERIMENTS = [f"C{i:03d}" for i in range(1, 12)] +VALID_RESOLUTIONS = [1, 2, 4, 8, 16] + + +def check_res(res): + """ + Validate the ISMIP7 grid resolution. + + Parameters + ---------- + res : str or int + Resolution in kilometres to validate. + + Raises + ------ + ValueError + If the resolution is not in the list of valid values. + """ + try: + res_int = int(res) + except (ValueError, TypeError): + raise ValueError( + f"Resolution '{res}' is not a valid integer. " + f"Valid resolutions (km) are: {VALID_RESOLUTIONS}") + if res_int not in VALID_RESOLUTIONS: + raise ValueError( + f"Invalid resolution '{res_int}' km. " + f"Valid resolutions (km) are: {VALID_RESOLUTIONS}") + print(f"Resolution {res_int} km is valid.") def check_exp_name(exp): diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 2c450b47f..211884cbd 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -1,11 +1,22 @@ #!/usr/bin/env python """ -This script processes MALI simulation outputs (both state and flux) -in the required format by the ISMIP7 experimental protocol. -The input state files (i.e., output files from MALI) need to have been -concatenated to have yearly data, which can be done using 'ncrcat' command -before using this script. +This script processes MALI simulation outputs into the required format +for submission to ISMIP7. +There are 3 flavors of variables that can be processed: +1D, 2D state, and 2D flux variables. +Any combination can be specified. +Processing assumes simulations were run with the output streams defined +by the ismip7_run compass test case. +Multifile output file sets should be specified with a glob pattern. +Note: wildcard paths should be quoted to avoid shell expansion. + +An ISMIP submimssion grid resolution and grid file need to be specified. +A mapping file will be created unless an existing one is specified. + +More info at: +https://github.com/ismip/ISM_SimulationChecker/blob/main/conventions/ISMIP7_variable_request.csv +https://www.ismip.org/research/ismip7 """ import argparse @@ -14,7 +25,7 @@ import glob from datetime import datetime from grid_and_mapping import build_mapping_file, check_ismip7_grid_file, \ - check_exp_name + check_exp_name, check_res from process_state_variables_ismip7 import generate_output_2d_state_vars, \ process_state_vars from process_1d_variables_ismip7 import generate_output_1d_vars, \ @@ -23,22 +34,22 @@ def main(): parser = argparse.ArgumentParser( - description='process MALI outputs for the ISMIP7' - 'submission') + description=__doc__) parser.add_argument("-e", "--exp_name", dest="exp", required=True, help="ISMIP7 experiment name (e.g., exp05") - parser.add_argument("-i_state", "--input_state", dest="input_file_state", + parser.add_argument("-s", "--input_state", dest="input_file_state", required=False, help="mpas output state variables") - parser.add_argument("-i_flux", "--input_flux", dest="input_file_flux", + parser.add_argument("-f", "--input_flux", dest="input_file_flux", required=False, help="mpas output flux variables") - parser.add_argument("-i_mesh", "--input_mesh", dest="input_file_grid", + parser.add_argument("-m", "--input_mesh", dest="input_file_mesh", required=False, help="MALI file with mesh information") parser.add_argument("-g", "--global_stats_pattern", dest="global_stats_pattern", required=False, help="glob pattern matching one or more globalStats.nc " - "files (e.g. 'globalStats_*.nc')") - parser.add_argument("-p", "--output_path", dest="output_path", + "files (e.g. 'globalStats_*.nc')." + "Note: wildcard paths should be quoted to avoid shell expansion.") + parser.add_argument("-o", "--output_path", dest="output_path", required=True, help="path to which the final output files" " will be saved") @@ -52,7 +63,7 @@ def main(): help="mapping method. Default='conserve'") parser.add_argument("--res", dest="res_ismip7_grid", required=True, - help="resolution of the ismip7 grid, (e.g. 8 for 8km res)") + help="resolution of the ismip7 grid, in kilometers: 16, 8, 4, 2, 1") parser.add_argument("--icesheet", dest="icesheet", required=True, choices=['AIS', 'GIS'], @@ -60,11 +71,14 @@ def main(): args = parser.parse_args() check_exp_name(args.exp) - check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) + check_res(args.res_ismip7_grid) + print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process if not args.input_file_state is None or not args.input_file_flux is None: + # Check grid file and res if 2d variables are to be processed + check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) # Check mapping method and either reuse an existing map or create a new one. # Note: the function 'building_mapping_file' requires the mpas mesh tool # script 'create_SCRIP_file_from_planar_rectangular_grid.py' @@ -76,7 +90,7 @@ def main(): print(f"Reusing existing mapping file: {args.reuse_mapping_file}") mapping_file = args.reuse_mapping_file else: - if args.input_file_grid is None: + if args.input_file_mesh is None: raise ValueError("--input_mesh is required when creating a new " "mapping file.") @@ -86,7 +100,7 @@ def main(): print(f"Creating new mapping file." f"Mapping method used: {method_remap}") - build_mapping_file(args.input_file_grid, mapping_file, + build_mapping_file(args.input_file_mesh, mapping_file, args.res_ismip7_grid, args.ismip7_grid_file, method_remap) From 63dd114e2cd7f8731db73d5c7aef30a4170c251b Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 10:36:27 -0700 Subject: [PATCH 17/33] Improve 1d state and flux variable time handling --- .../process_1d_variables_ismip7.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index 3d22c9a6f..e0e642e93 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -277,21 +277,31 @@ def generate_output_1d_vars(files, exp, icesheet, output_path=None): # this is for the state variables for i in range(nt_state): - ind_snap = np.where(decYears==years_state[i])[0] - - vol_snapshot[i] = vol[ind_snap] - vaf_snapshot[i] = vaf[ind_snap] - gia_snapshot[i] = gia[ind_snap] - fia_snapshot[i] = fia[ind_snap] - days_snapshot[i] = daysSinceStart[ind_snap] - - if decYears[ind_snap] == endYr: + # Use isclose to avoid floating-point equality issues. + ind_snap = np.where(np.isclose(decYears, years_state[i]))[0] + if len(ind_snap) == 0: + raise ValueError(f"No state snapshot found for year {years_state[i]}.") + if len(ind_snap) > 1: + print(f"WARNING: Found {len(ind_snap)} snapshots for year " + f"{years_state[i]}; using the first one.") + idx_snap = ind_snap[0] + + vol_snapshot[i] = vol[idx_snap] + vaf_snapshot[i] = vaf[idx_snap] + gia_snapshot[i] = gia[idx_snap] + fia_snapshot[i] = fia[idx_snap] + days_snapshot[i] = daysSinceStart[idx_snap] + + if decYears[idx_snap] == endYr: break # this is for the flux variables for i in range(nt_flux): ind_avg = np.where(np.logical_and(decYears > years_flux[i], decYears <= (years_flux[i] + 1.0)))[0] + if len(ind_avg) == 0: + raise ValueError(f"No flux averaging samples found for year " + f"{years_flux[i]}.") smbi = smb[ind_avg] bmbGri = bmbGr[ind_avg] bmbFlti = bmbFlt[ind_avg] From 73d9ec60d7ed305ca03bf120cd259a3b81d9f60b Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 10:49:04 -0700 Subject: [PATCH 18/33] Improve metadata handling * allow authors and group to be input * created single metadata dict for all stages to use --- .../post_process_mali_to_ismip7.py | 28 ++++++++-- .../process_1d_variables_ismip7.py | 45 +++++++--------- .../process_flux_variables_ismip7.py | 31 ++++++----- .../process_state_variables_ismip7.py | 51 +++++++++---------- 4 files changed, 82 insertions(+), 73 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 211884cbd..4c1a2b937 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -32,6 +32,10 @@ check_global_stats_files from process_flux_variables_ismip7 import generate_output_2d_flux_vars +DEFAULT_AUTHORS = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' +DEFAULT_GROUP = 'Los Alamos National Laboratory, Department of Energy' +DEFAULT_MODEL = 'MALI (MPAS-Albany Land Ice model)' + def main(): parser = argparse.ArgumentParser( description=__doc__) @@ -68,11 +72,28 @@ def main(): required=True, choices=['AIS', 'GIS'], help="ice sheet domain: 'AIS' (Antarctica) or 'GIS' (Greenland)") + parser.add_argument("--authors", dest="authors", + required=False, default=DEFAULT_AUTHORS, + help=f"author string for output file metadata " + f"(default: '{DEFAULT_AUTHORS}')") + parser.add_argument("--group", dest="group", + required=False, default=DEFAULT_GROUP, + help=f"group/institution string for output file metadata " + f"(default: '{DEFAULT_GROUP}')") args = parser.parse_args() check_exp_name(args.exp) check_res(args.res_ismip7_grid) + metadata = { + 'exp': args.exp, + 'icesheet': args.icesheet, + 'authors': args.authors, + 'group': args.group, + 'model': DEFAULT_MODEL, + 'date': datetime.now().strftime("%d-%b-%Y"), + } + print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process @@ -119,8 +140,7 @@ def main(): print("\n---Processing global stats file(s)---") global_stats_files = sorted(glob.glob(args.global_stats_pattern)) check_global_stats_files(global_stats_files) - generate_output_1d_vars(global_stats_files, args.exp, args.icesheet, - output_path) + generate_output_1d_vars(global_stats_files, output_path, metadata) print("---Processing global stats file(s) complete---\n") # process 2d state variables @@ -150,7 +170,7 @@ def main(): print("Writing processed and remapped state fields to ISMIP7 file format.") generate_output_2d_state_vars(processed_and_remapped_file_state, args.ismip7_grid_file, - args.exp, output_path, args.icesheet) + output_path, metadata) os.remove(tmp_file) os.remove(processed_and_remapped_file_state) @@ -175,7 +195,7 @@ def main(): # write out the output files in the ismip7-required format generate_output_2d_flux_vars(processed_file_flux, args.ismip7_grid_file, - args.exp, output_path, args.icesheet) + output_path, metadata) cleanUp = True if cleanUp: diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index e0e642e93..b5b9209c1 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -94,11 +94,10 @@ def check_global_stats_files(files): def _write_state_var(varname, data_values, time_days, standard_name, units, - variable_desc, output_path, simulationStartDate, - author_str, date_str, exp, icesheet): + variable_desc, output_path, simulationStartDate, metadata): """Write a single 1D state (snapshot) variable to a NETCDF4_CLASSIC file.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_{icesheet}_DOE_MALI_{exp}.nc', + ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) var_out = ds_out.createVariable(varname, 'd', ('time',)) @@ -111,20 +110,20 @@ def _write_state_var(varname, data_values, time_days, standard_name, units, time_out.long_name = 'time' var_out.standard_name = standard_name var_out.units = units - ds_out.AUTHORS = author_str - ds_out.MODEL = 'MALI (MPAS-Albany Land Ice model)' - ds_out.GROUP = 'Los Alamos National Laboratory' + ds_out.AUTHORS = metadata['authors'] + ds_out.MODEL = metadata['model'] + ds_out.GROUP = metadata['group'] ds_out.VARIABLE = variable_desc - ds_out.DATE = date_str + ds_out.DATE = metadata['date'] ds_out.close() def _write_flux_var(varname, data_values, days_min, days_max, standard_name, units, variable_desc, output_path, simulationStartDate, - author_str, date_str, exp, icesheet): + metadata): """Write a single 1D flux (time-averaged) variable to a NETCDF4_CLASSIC file.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_{icesheet}_DOE_MALI_{exp}.nc', + ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) ds_out.createDimension('bnds', 2) @@ -141,15 +140,15 @@ def _write_flux_var(varname, data_values, days_min, days_max, standard_name, time_out.long_name = 'time' var_out.standard_name = standard_name var_out.units = units - ds_out.AUTHORS = author_str - ds_out.MODEL = 'MALI (MPAS-Albany Land Ice model)' - ds_out.GROUP = 'Los Alamos National Laboratory' + ds_out.AUTHORS = metadata['authors'] + ds_out.MODEL = metadata['model'] + ds_out.GROUP = metadata['group'] ds_out.VARIABLE = variable_desc - ds_out.DATE = date_str + ds_out.DATE = metadata['date'] ds_out.close() -def generate_output_1d_vars(files, exp, icesheet, output_path=None): +def generate_output_1d_vars(files, output_path, metadata): """ Process and write 1D (scalar time-series) state and flux variables. @@ -157,20 +156,15 @@ def generate_output_1d_vars(files, exp, icesheet, output_path=None): ---------- files : list of str Sorted list of globalStats.nc file paths to process. - exp : str - ISMIP7 experiment name (e.g. 'C001'). - icesheet : str - Ice sheet domain, either 'AIS' or 'GIS'. - output_path : str, optional - Directory for output files. Defaults to current working directory. + output_path : str + Directory for output files. + metadata : dict + Submission metadata with keys: exp, icesheet, authors, group, model, date. """ if output_path is None or not os.path.exists(output_path): output_path = os.getcwd() - AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' - DATE_STR = date.today().strftime("%d-%b-%Y") - ds = xr.open_mfdataset(files, combine='nested', concat_dim='Time', decode_cf=False, data_vars='minimal', coords='minimal', compat='override') @@ -327,10 +321,7 @@ def generate_output_1d_vars(files, exp, icesheet, output_path=None): common = dict( output_path=output_path, simulationStartDate=simulationStartDate, - author_str=AUTHOR_STR, - date_str=DATE_STR, - exp=exp, - icesheet=icesheet, + metadata=metadata, ) # --- state (snapshot) variables --- diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index 153d1b54a..1b6a8d121 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -14,7 +14,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, var_units, var_varname, remapped_mali_flux_file, - ismip7_grid_file, exp, output_path): + ismip7_grid_file, output_path, metadata): """ mali_var_name: variable name on MALI side @@ -47,7 +47,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{icesheet}_DOE_MALI_{exp}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('bnds', 2) @@ -61,7 +61,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, timeValues = dataOut.createVariable('time', 'd', ('time')) AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' - DATE_STR = date.today().strftime("%d-%b-%Y") + DATE_STR = metadata['date'] for i in range(timeSteps): mask = iceMask[i, :, :] @@ -93,18 +93,17 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, yValues.units = 'm' yValues.standard_name = 'y' yValues.long_name = 'y' - dataOut.AUTHORS = AUTHOR_STR - dataOut.MODEL = 'MALI (MPAS-Albany Land Ice model)' - dataOut.GROUP = 'Los Alamos National Laboratory, Department of Energy' + dataOut.AUTHORS = metadata['authors'] + dataOut.MODEL = metadata['model'] + dataOut.GROUP = metadata['group'] dataOut.VARIABLE = var_varname - dataOut.DATE = DATE_STR + dataOut.DATE = metadata['date'] dataOut.close() data.close() def generate_output_2d_flux_vars(file_remapped_mali_flux, - ismip7_grid_file, exp, output_path, - icesheet): + ismip7_grid_file, output_path, metadata): """ file_remapped_mali_flux: flux output file on mali mesh remapped onto the ismip7 grid @@ -120,7 +119,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'land_ice_surface_specific_mass_balance_flux', 'kg m-2 s-1', 'Surface mass balance flux', file_remapped_mali_flux, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- libmassbffl ------------------ write_netcdf_2d_flux_vars('avgFloatingBMBFlux', 'libmassbffl', @@ -128,7 +127,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Basal mass balance flux beneath floating ice', file_remapped_mali_flux, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- libmassbfgr ------------------ write_netcdf_2d_flux_vars('avgGroundedBMBFlux', 'libmassbfgr', @@ -136,7 +135,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Basal mass balance flux beneath grounded ice', file_remapped_mali_flux, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- dlithkdt ------------------ write_netcdf_2d_flux_vars('avgDhdt', 'dlithkdt', @@ -144,7 +143,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'm s-1', 'Ice thickness imbalance', file_remapped_mali_flux, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- licalvf ------------------ write_netcdf_2d_flux_vars('avgCalvingFlux', 'licalvf', @@ -152,7 +151,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Calving flux', file_remapped_mali_flux, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- lifmassbf ------------------ # Note: facemelting and calving flux are combined above @@ -161,7 +160,7 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Ice front melt flux', file_remapped_mali_flux, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- ligroundf ------------------ write_netcdf_2d_flux_vars('avgGroundingLineFlux', 'ligroundf', @@ -169,4 +168,4 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'kg m-2 s-1', 'Grounding line flux', file_remapped_mali_flux, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 0e84ab122..445f2d134 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -60,7 +60,7 @@ def process_state_vars(inputfile_state, tmp_file): def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, var_units, var_varname, remapped_mali_outputfile, - ismip7_grid_file, exp, output_path): + ismip7_grid_file, output_path, metadata): """ mali_var_name: variable name on MALI side ismip7_var_name: variable name required by ISMIP7 @@ -89,7 +89,7 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{icesheet}_DOE_MALI_{exp}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('x', lonN) @@ -101,7 +101,7 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, timeValues = dataOut.createVariable('time', 'd', ('time')) timeValues[:] = daysSinceStart AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' - DATE_STR = date.today().strftime("%d-%b-%Y") + DATE_STR = metadata['date'] for i in range(timeSteps): if ismip7_var_name == 'sftgif': @@ -137,17 +137,16 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, yValues.units = 'm' yValues.standard_name = 'y' yValues.long_name = 'y' - dataOut.AUTHORS = AUTHOR_STR - dataOut.MODEL = 'MALI (MPAS-Albany Land Ice model)' - dataOut.GROUP = 'Los Alamos National Laboratory, Department of Energy' + dataOut.AUTHORS = metadata['authors'] + dataOut.MODEL = metadata['model'] + dataOut.GROUP = metadata['group'] dataOut.VARIABLE = var_varname - dataOut.DATE = DATE_STR + dataOut.DATE = metadata['date'] dataOut.close() def generate_output_2d_state_vars(file_remapped_mali_state, - ismip7_grid_file, exp, output_path, - icesheet): + ismip7_grid_file, output_path, metadata): """ file_remapped_mali_state: output files on mali mesh remapped on the ismip7 grid @@ -161,25 +160,25 @@ def generate_output_2d_state_vars(file_remapped_mali_state, write_netcdf_2d_state_vars('thickness','lithk', 'land_ice_thickness', 'm', 'Ice thickness', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- orog ------------------ write_netcdf_2d_state_vars('upperSurface','orog', 'surface_altitude', 'm', 'Surface elevation', file_remapped_mali_state, - ismip7_grid_file,exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- base ------------------ write_netcdf_2d_state_vars('lowerSurface','base', 'base_altitude', 'm', 'Base elevation', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- topg ------------------ write_netcdf_2d_state_vars('bedTopography','topg', 'bedrock_altitude', 'm', 'Bedrock elevation', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- hfgeoubed------------------ # Note: even though this is a flux variable, we are taking a snapshot of it @@ -195,28 +194,28 @@ def generate_output_2d_state_vars(file_remapped_mali_state, 'land_ice_surface_x_velocity', 'm s-1', 'Surface velocity in x', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # -----------yxvelsurf ------------------ write_netcdf_2d_state_vars('uReconstructY_sfc', 'yvelsurf', 'land_ice_surface_y_velocity', 'm s-1', 'Surface velocity in x', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- xvelbase ------------------ write_netcdf_2d_state_vars('uReconstructX_base', 'xvelbase', 'land_ice_basal_x_velocity', 'm s-1', 'Basal velocity in x', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- yvelbase ------------------ write_netcdf_2d_state_vars('uReconstructY_base', 'yvelbase', 'land_ice_basal_y_velocity', 'm s-1', 'Basal velocity in y', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- zvelsurf & zvelbase ------------------ # ISMIP7 requires these variables, but MALI does not output them. @@ -227,60 +226,60 @@ def generate_output_2d_state_vars(file_remapped_mali_state, 'land_ice_vertical_mean_x_velocity', 'm s-1', 'Mean velocity in x', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- yvelmean ------------------ write_netcdf_2d_state_vars('yvelmean', 'yvelmean', 'land_ice_vertical_mean_y_velocity', 'm s-1', 'Mean velocity in y', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- litemptop ------------------ write_netcdf_2d_state_vars('surfaceTemperature', 'litemptop', 'temperature_at_top_of_ice_sheet_model', 'K', 'Surface temperature', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- litempbotgr ------------------ write_netcdf_2d_state_vars('litempbotgr', 'litempbotgr', 'temperature_at_base_of_ice_sheet_model', 'K', 'Basal temperature beneath grounded ice sheet', file_remapped_mali_state, - ismip7_grid_file,exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- litempbotfl ------------------ write_netcdf_2d_state_vars('litempbotfl', 'litempbotfl', 'temperature_at_base_of_ice_sheet_model', 'K', 'Basal temperature beneath floating ice shelf', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- strbasemag ------------------ write_netcdf_2d_state_vars('strbasemag', 'strbasemag', 'land_ice_basal_drag ', 'Pa', 'Basal drag', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- sftgif ------------------ write_netcdf_2d_state_vars('sftgif','sftgif', 'land_ice_area_fraction', '1', 'Land ice area fraction', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- sftgrf ------------------ write_netcdf_2d_state_vars('sftgrf', 'sftgrf', 'grounded_ice_sheet_area_fraction', '1', 'Grounded ice sheet area fraction', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) # ----------- sftflf ------------------ write_netcdf_2d_state_vars('sftflf','sftflf', 'floating_ice_shelf_area_fraction', '1', 'Floating ice shelf area fraction', file_remapped_mali_state, - ismip7_grid_file, exp, output_path) + ismip7_grid_file, output_path, metadata) From e8649e5d518a7ce8663ee0b73f211ec0c1138277 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 10:53:13 -0700 Subject: [PATCH 19/33] Make group nickname in output filenames a CLA This makes it easy to set for DOE and Arete submissions --- .../ismip7_postprocessing/post_process_mali_to_ismip7.py | 6 ++++++ .../ismip7_postprocessing/process_1d_variables_ismip7.py | 4 ++-- .../ismip7_postprocessing/process_flux_variables_ismip7.py | 2 +- .../ismip7_postprocessing/process_state_variables_ismip7.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 4c1a2b937..b61838c65 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -35,6 +35,7 @@ DEFAULT_AUTHORS = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' DEFAULT_GROUP = 'Los Alamos National Laboratory, Department of Energy' DEFAULT_MODEL = 'MALI (MPAS-Albany Land Ice model)' +DEFAULT_GROUP_NICKNAME = 'DOE' def main(): parser = argparse.ArgumentParser( @@ -80,6 +81,10 @@ def main(): required=False, default=DEFAULT_GROUP, help=f"group/institution string for output file metadata " f"(default: '{DEFAULT_GROUP}')") + parser.add_argument("--group_nickname", dest="group_nickname", + required=False, default=DEFAULT_GROUP_NICKNAME, + help=f"short group nickname used in output filenames " + f"(default: '{DEFAULT_GROUP_NICKNAME}')") args = parser.parse_args() check_exp_name(args.exp) @@ -90,6 +95,7 @@ def main(): 'icesheet': args.icesheet, 'authors': args.authors, 'group': args.group, + 'group_nickname': args.group_nickname, 'model': DEFAULT_MODEL, 'date': datetime.now().strftime("%d-%b-%Y"), } diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index b5b9209c1..197c1e7ac 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -97,7 +97,7 @@ def _write_state_var(varname, data_values, time_days, standard_name, units, variable_desc, output_path, simulationStartDate, metadata): """Write a single 1D state (snapshot) variable to a NETCDF4_CLASSIC file.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', + ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) var_out = ds_out.createVariable(varname, 'd', ('time',)) @@ -123,7 +123,7 @@ def _write_flux_var(varname, data_values, days_min, days_max, standard_name, metadata): """Write a single 1D flux (time-averaged) variable to a NETCDF4_CLASSIC file.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', + ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) ds_out.createDimension('bnds', 2) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index 1b6a8d121..ecab769d6 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -47,7 +47,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('bnds', 2) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 445f2d134..eb2a71bd7 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -89,7 +89,7 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_DOE_MALI_{metadata["exp"]}.nc', + dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', 'w', format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('x', lonN) From 1c4f602b8d42ca6ecb57d0b168f59ee73fa20d38 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 11:02:17 -0700 Subject: [PATCH 20/33] Convert 2d state vars to multifile dataset Move multifile validation to a helper module and call from both 1d and 2d state processing. --- .../post_process_mali_to_ismip7.py | 24 +++--- .../process_1d_variables_ismip7.py | 67 +-------------- .../process_state_variables_ismip7.py | 40 ++++++--- .../ismip7_postprocessing/validate.py | 83 +++++++++++++++++++ 4 files changed, 129 insertions(+), 85 deletions(-) create mode 100644 landice/output_processing_li/ismip7_postprocessing/validate.py diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index b61838c65..84d179ab7 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -27,7 +27,7 @@ from grid_and_mapping import build_mapping_file, check_ismip7_grid_file, \ check_exp_name, check_res from process_state_variables_ismip7 import generate_output_2d_state_vars, \ - process_state_vars + process_state_vars, check_state_files from process_1d_variables_ismip7 import generate_output_1d_vars, \ check_global_stats_files from process_flux_variables_ismip7 import generate_output_2d_flux_vars @@ -43,8 +43,10 @@ def main(): parser.add_argument("-e", "--exp_name", dest="exp", required=True, help="ISMIP7 experiment name (e.g., exp05") - parser.add_argument("-s", "--input_state", dest="input_file_state", - required=False, help="mpas output state variables") + parser.add_argument("-s", "--input_state_pattern", dest="input_state_pattern", + required=False, + help="glob pattern matching one or more MALI state output files. " + "Note: wildcard paths should be quoted to avoid shell expansion.") parser.add_argument("-f", "--input_flux", dest="input_file_flux", required=False, help="mpas output flux variables") parser.add_argument("-m", "--input_mesh", dest="input_file_mesh", @@ -103,7 +105,7 @@ def main(): print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process - if not args.input_file_state is None or not args.input_file_flux is None: + if not args.input_state_pattern is None or not args.input_file_flux is None: # Check grid file and res if 2d variables are to be processed check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) # Check mapping method and either reuse an existing map or create a new one. @@ -150,19 +152,19 @@ def main(): print("---Processing global stats file(s) complete---\n") # process 2d state variables - if args.input_file_state is None: - print("--- MALI state file is not provided, thus it will not be processed.") + if args.input_state_pattern is None: + print("--- MALI state pattern is not provided, thus it will not be processed.") else: - print("\n---Processing state file---") - # state variables processing part + print("\n---Processing state file(s)---") + state_files = sorted(glob.glob(args.input_state_pattern)) + check_state_files(state_files) # process (add and rename) state vars as requested by the ISMIP7 protocol print("Calculating needed state file adjustments.") tmp_file = "tmp_state.nc" - process_state_vars(args.input_file_state, tmp_file) + process_state_vars(state_files, tmp_file) # remap data from the MALI unstructured mesh to the ISMIP7 polarstereo grid - processed_and_remapped_file_state = f'processed_and_remapped_' \ - f'{os.path.basename(args.input_file_state)}' + processed_and_remapped_file_state = 'processed_and_remapped_state.nc' print("Remapping state file.") command = ["ncremap", diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index 197c1e7ac..0a769670d 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -10,6 +10,7 @@ import sys import glob import xarray as xr +from validate import validate_mali_files EXPECTED_VARIABLES = [ @@ -26,71 +27,9 @@ def check_global_stats_files(files): """ Validate a list of globalStats files before processing. - Checks that: - - The list is not empty - - Each file exists - - Each file contains the expected variables - - simulationStartTime is consistent across all files - - No time overlaps exist between consecutive files - - No unexpectedly large time gaps (> 366 days) exist between consecutive files - - Parameters - ---------- - files : list of str - Sorted list of globalStats file paths. - - Raises - ------ - ValueError - If any validation check fails. - FileNotFoundError - If any file does not exist. + See validate.validate_mali_files for full details of checks performed. """ - if len(files) == 0: - raise ValueError( - "No globalStats files matched the provided glob pattern.") - - for f in files: - if not os.path.exists(f): - raise FileNotFoundError(f"globalStats file not found: {f}") - - # Check required variables in each file - for f in files: - with xr.open_dataset(f, decode_cf=False) as ds: - missing = [v for v in EXPECTED_VARIABLES if v not in ds] - if missing: - raise ValueError( - f"File '{f}' is missing expected variables: {missing}") - - # Check simulationStartTime consistency across all files - start_times = [] - for f in files: - with xr.open_dataset(f, decode_cf=False) as ds: - start_times.append( - ds['simulationStartTime'].values.tobytes() - .decode('utf-8').strip().strip('\x00')) - if len(set(start_times)) > 1: - raise ValueError( - f"Inconsistent simulationStartTime across globalStats files: " - f"{set(start_times)}") - - # Check for time overlaps or large gaps between consecutive files - for i in range(len(files) - 1): - with xr.open_dataset(files[i], decode_cf=False) as ds_a: - end_a = float(ds_a['daysSinceStart'].values[-1]) - with xr.open_dataset(files[i + 1], decode_cf=False) as ds_b: - start_b = float(ds_b['daysSinceStart'].values[0]) - if start_b <= end_a: - raise ValueError( - f"Time overlap detected between files:\n" - f" {files[i]} (ends at day {end_a})\n" - f" {files[i + 1]} (starts at day {start_b})") - gap_days = start_b - end_a - if gap_days > 366: - print(f"WARNING: Gap of {gap_days:.1f} days between files:\n" - f" {files[i]}\n {files[i + 1]}") - - print(f"Validated {len(files)} globalStats file(s).") + validate_mali_files(files, EXPECTED_VARIABLES, label='globalStats') def _write_state_var(varname, data_values, time_days, standard_name, units, diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index eb2a71bd7..611c7fb1a 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -1,28 +1,48 @@ """ This script has functions that are needed to post-process and write state output variables from ISMIP7 simulations. -The input files (i.e., MALI output files) need to have been -concatenated to have yearly data, which can be done using 'ncrcat' command -before using this script. """ from netCDF4 import Dataset import xarray as xr import numpy as np from datetime import date -import shutil import os, sys +from validate import validate_mali_files -def process_state_vars(inputfile_state, tmp_file): +EXPECTED_STATE_VARIABLES = [ + 'daysSinceStart', 'simulationStartTime', + 'cellMask', 'basalTemperature', 'betaSolve', + 'uReconstructX', 'uReconstructY', 'upperSurface', +] + + +def check_state_files(files): + """ + Validate a list of MALI state output files before processing. + + See validate.validate_mali_files for full details of checks performed. + """ + validate_mali_files(files, EXPECTED_STATE_VARIABLES, label='state') + +def process_state_vars(files, tmp_file): """ - inputfile_state: output file copy from MALI simulations - tmp_file: temporary file name - inputfile_temperature: output temperature file from MALI simulations + files: list of MALI state output file paths + tmp_file: temporary output file name """ - inputfile_state_vars = xr.open_dataset(inputfile_state, engine="netcdf4", decode_cf=False) - del inputfile_state_vars.daysSinceStart.attrs['units'] # need this line to prevent xarray from reading daysSinceStart as a timedelta type and corrupting values after about 250 years + inputfile_state_vars = xr.open_mfdataset(files, combine='nested', + concat_dim='Time', + engine='netcdf4', + decode_cf=False, + data_vars='minimal', + coords='minimal', + compat='override') + # Delete the 'units' attr on daysSinceStart to prevent xarray from + # encoding it as a timedelta type (corrupts values after ~250 years). + if 'units' in inputfile_state_vars['daysSinceStart'].attrs: + del inputfile_state_vars['daysSinceStart'].attrs['units'] # get the mesh description data nCells = inputfile_state_vars.dims['nCells'] diff --git a/landice/output_processing_li/ismip7_postprocessing/validate.py b/landice/output_processing_li/ismip7_postprocessing/validate.py new file mode 100644 index 000000000..9dadb1930 --- /dev/null +++ b/landice/output_processing_li/ismip7_postprocessing/validate.py @@ -0,0 +1,83 @@ +""" +Generic validation utilities for sets of MALI output files. +""" + +import os +import xarray as xr + + +def validate_mali_files(files, required_vars, label=''): + """ + Validate a sorted list of MALI output files before processing. + + Checks that: + - The list is not empty + - Each file exists + - Each file contains all required variables + - simulationStartTime is consistent across all files + - No time overlaps (daysSinceStart) exist between consecutive files + - No unexpectedly large time gaps (> 366 days) exist between consecutive files + + Parameters + ---------- + files : list of str + Sorted list of file paths. + required_vars : list of str + Variable names that must be present in every file. + label : str, optional + Human-readable label for the file type, used in messages. + + Raises + ------ + ValueError + If any validation check fails. + FileNotFoundError + If any file does not exist. + """ + tag = f' ({label})' if label else '' + + if len(files) == 0: + raise ValueError(f"No files provided{tag}.") + + for f in files: + if not os.path.exists(f): + raise FileNotFoundError(f"File not found{tag}: {f}") + + # Check required variables in each file + for f in files: + with xr.open_dataset(f, decode_cf=False) as ds: + missing = [v for v in required_vars if v not in ds] + if missing: + raise ValueError( + f"File '{f}'{tag} is missing expected variables: {missing}") + + # Check simulationStartTime consistency across all files + if len(files) > 1: + start_times = [] + for f in files: + with xr.open_dataset(f, decode_cf=False) as ds: + start_times.append( + ds['simulationStartTime'].values.tobytes() + .decode('utf-8').strip().strip('\x00')) + if len(set(start_times)) > 1: + raise ValueError( + f"Inconsistent simulationStartTime across files{tag}: " + f"{set(start_times)}") + + # Check for time overlaps or large gaps between consecutive files + for i in range(len(files) - 1): + with xr.open_dataset(files[i], decode_cf=False) as ds_a: + end_a = float(ds_a['daysSinceStart'].values[-1]) + with xr.open_dataset(files[i + 1], decode_cf=False) as ds_b: + start_b = float(ds_b['daysSinceStart'].values[0]) + if start_b <= end_a: + raise ValueError( + f"Time overlap detected between files{tag}:\n" + f" {files[i]} (ends at day {end_a})\n" + f" {files[i + 1]} (starts at day {start_b})") + gap_days = start_b - end_a + if gap_days > 366: + print(f"WARNING: Gap of {gap_days:.1f} days between files{tag}:\n" + f" {files[i]}\n {files[i + 1]}") + + print(f"Validated {len(files)} file(s){tag}.") From 6e29f017bd9f7db8b34dfa9991394247c51dcb9f Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 11:10:23 -0700 Subject: [PATCH 21/33] move state processing steps out of main script --- .../post_process_mali_to_ismip7.py | 32 ++------------- .../process_state_variables_ismip7.py | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 84d179ab7..c2a618834 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -26,8 +26,7 @@ from datetime import datetime from grid_and_mapping import build_mapping_file, check_ismip7_grid_file, \ check_exp_name, check_res -from process_state_variables_ismip7 import generate_output_2d_state_vars, \ - process_state_vars, check_state_files +from process_state_variables_ismip7 import process_state_pipeline from process_1d_variables_ismip7 import generate_output_1d_vars, \ check_global_stats_files from process_flux_variables_ismip7 import generate_output_2d_flux_vars @@ -157,32 +156,9 @@ def main(): else: print("\n---Processing state file(s)---") state_files = sorted(glob.glob(args.input_state_pattern)) - check_state_files(state_files) - # process (add and rename) state vars as requested by the ISMIP7 protocol - print("Calculating needed state file adjustments.") - tmp_file = "tmp_state.nc" - process_state_vars(state_files, tmp_file) - - # remap data from the MALI unstructured mesh to the ISMIP7 polarstereo grid - processed_and_remapped_file_state = 'processed_and_remapped_state.nc' - - print("Remapping state file.") - command = ["ncremap", - "-i", tmp_file, - "-o", processed_and_remapped_file_state, - "-m", mapping_file, - "-P", "mpas"] - check_call(command) - - # write out 2D state output files in the ismip7-required format - print("Writing processed and remapped state fields to ISMIP7 file format.") - generate_output_2d_state_vars(processed_and_remapped_file_state, - args.ismip7_grid_file, - output_path, metadata) - - os.remove(tmp_file) - os.remove(processed_and_remapped_file_state) - print("---Processing state file complete---\n") + process_state_pipeline(state_files, mapping_file, args.ismip7_grid_file, + output_path, metadata) + print("---Processing state file(s) complete---\n") # process 2d flux variables if args.input_file_flux is None: diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 611c7fb1a..6896641ce 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -8,6 +8,7 @@ import numpy as np from datetime import date import os, sys +from subprocess import check_call from validate import validate_mali_files @@ -165,6 +166,46 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, dataOut.close() +def process_state_pipeline(state_files, mapping_file, ismip7_grid_file, + output_path, metadata): + """ + Full state-variable processing pipeline: validate, adjust, remap, write. + + Parameters + ---------- + state_files : list of str + Sorted list of MALI state output file paths. + mapping_file : str + Path to the ESMF mapping/weights file. + ismip7_grid_file : str + Path to the ISMIP7 grid file. + output_path : str + Directory for output files. + metadata : dict + Submission metadata (exp, icesheet, authors, group, model, date, ...). + """ + check_state_files(state_files) + + print("Calculating needed state file adjustments.") + tmp_file = "tmp_state.nc" + process_state_vars(state_files, tmp_file) + + processed_and_remapped_file_state = 'processed_and_remapped_state.nc' + print("Remapping state file.") + check_call(["ncremap", + "-i", tmp_file, + "-o", processed_and_remapped_file_state, + "-m", mapping_file, + "-P", "mpas"]) + + print("Writing processed and remapped state fields to ISMIP7 file format.") + generate_output_2d_state_vars(processed_and_remapped_file_state, + ismip7_grid_file, output_path, metadata) + + os.remove(tmp_file) + os.remove(processed_and_remapped_file_state) + + def generate_output_2d_state_vars(file_remapped_mali_state, ismip7_grid_file, output_path, metadata): """ From 223073bd668e4cf9a02091f2bab8426dea8346a8 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 12:56:44 -0700 Subject: [PATCH 22/33] update scrip command to that of compass env --- .../ismip7_postprocessing/grid_and_mapping.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py index 04369a580..e2a69f60a 100644 --- a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py +++ b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py @@ -164,7 +164,7 @@ def build_mapping_file(mali_mesh_file, f"for ismip7 grid and mali mesh...") # create a scripfile for ismip7 grid - args = ["create_SCRIP_file_from_planar_rectangular_grid.py", + args = ["create_scrip_file_from_planar_rectangular_grid", "--input", ismip7_grid_file, "--scrip", ismip7_scripfile, "--proj", ismip7_projection, @@ -204,6 +204,7 @@ def build_mapping_file(mali_mesh_file, "-i", "-64bit_offset", "--dst_regional", "--src_regional"]) + print(f"Running remapping command: {' '.join(args)}") check_call(args) # remove the temporary scripfiles once the mapping file is generated From de731ed461ecf4f34bbeab77a5df1a87fae4c1b6 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 12:57:48 -0700 Subject: [PATCH 23/33] Apply mfdataset to flux vars --- .../post_process_mali_to_ismip7.py | 40 +++------ .../process_flux_variables_ismip7.py | 83 +++++++++++++++++++ 2 files changed, 96 insertions(+), 27 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index c2a618834..b5a031aa2 100755 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -29,7 +29,7 @@ from process_state_variables_ismip7 import process_state_pipeline from process_1d_variables_ismip7 import generate_output_1d_vars, \ check_global_stats_files -from process_flux_variables_ismip7 import generate_output_2d_flux_vars +from process_flux_variables_ismip7 import process_flux_pipeline DEFAULT_AUTHORS = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' DEFAULT_GROUP = 'Los Alamos National Laboratory, Department of Energy' @@ -46,8 +46,10 @@ def main(): required=False, help="glob pattern matching one or more MALI state output files. " "Note: wildcard paths should be quoted to avoid shell expansion.") - parser.add_argument("-f", "--input_flux", dest="input_file_flux", - required=False, help="mpas output flux variables") + parser.add_argument("-f", "--input_flux_pattern", dest="input_flux_pattern", + required=False, + help="glob pattern matching one or more MALI flux output files. " + "Note: wildcard paths should be quoted to avoid shell expansion.") parser.add_argument("-m", "--input_mesh", dest="input_file_mesh", required=False, help="MALI file with mesh information") parser.add_argument("-g", "--global_stats_pattern", dest="global_stats_pattern", @@ -104,7 +106,7 @@ def main(): print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process - if not args.input_state_pattern is None or not args.input_file_flux is None: + if not args.input_state_pattern is None or not args.input_flux_pattern is None: # Check grid file and res if 2d variables are to be processed check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) # Check mapping method and either reuse an existing map or create a new one. @@ -161,30 +163,14 @@ def main(): print("---Processing state file(s) complete---\n") # process 2d flux variables - if args.input_file_flux is None: - print("--- MALI flux file is not provided, thus it will not be processed.") + if args.input_flux_pattern is None: + print("--- MALI flux pattern is not provided, thus it will not be processed.") else: - print("\n---Processing flux file---") - - # remap data from the MALI unstructured mesh to the ISMIP7 P-S grid - processed_file_flux = f'processed_' \ - f'{os.path.basename(args.input_file_flux)}' - command = ["ncremap", - "-i", args.input_file_flux, - "-o", processed_file_flux, - "-m", mapping_file, - "-P", "mpas"] - check_call(command) - - # write out the output files in the ismip7-required format - generate_output_2d_flux_vars(processed_file_flux, - args.ismip7_grid_file, - output_path, metadata) - - cleanUp = True - if cleanUp: - os.remove(processed_file_flux) - print("---Processing flux file complete---\n") + print("\n---Processing flux file(s)---") + flux_files = sorted(glob.glob(args.input_flux_pattern)) + process_flux_pipeline(flux_files, mapping_file, args.ismip7_grid_file, + output_path, metadata) + print("---Processing flux file(s) complete---\n") print("---All processing complete---") if __name__ == "__main__": diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index ecab769d6..448b82b9a 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -10,6 +10,48 @@ from subprocess import check_call import os, sys import warnings +from validate import validate_mali_files + + +EXPECTED_FLUX_VARIABLES = [ + 'daysSinceStart', 'simulationStartTime', + 'timeBndsMin', 'timeBndsMax', 'iceMask', + 'avgSMBFlux', 'avgFloatingBMBFlux', 'avgGroundedBMBFlux', + 'avgDhdt', 'avgCalvingFlux', 'avgGroundingLineFlux', +] + + +def check_flux_files(files): + """ + Validate a list of MALI flux output files before processing. + + See validate.validate_mali_files for full details of checks performed. + """ + validate_mali_files(files, EXPECTED_FLUX_VARIABLES, label='flux') + + +def process_flux_vars(files, tmp_file): + """ + Concatenate/prepare flux files into a temporary file for remapping. + + Parameters + ---------- + files : list of str + Sorted list of MALI flux output file paths. + tmp_file : str + Temporary output file name. + """ + ds_flux = xr.open_mfdataset(files, combine='nested', + concat_dim='Time', + engine='netcdf4', + decode_cf=False, + data_vars='minimal', + coords='minimal', + compat='override') + if 'daysSinceStart' in ds_flux and 'units' in ds_flux['daysSinceStart'].attrs: + del ds_flux['daysSinceStart'].attrs['units'] + ds_flux.to_netcdf(tmp_file) + ds_flux.close() def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, @@ -169,3 +211,44 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, 'Grounding line flux', file_remapped_mali_flux, ismip7_grid_file, output_path, metadata) + + +def process_flux_pipeline(flux_files, mapping_file, ismip7_grid_file, + output_path, metadata): + """ + Full flux-variable processing pipeline: validate, concatenate, remap, write. + + Parameters + ---------- + flux_files : list of str + Sorted list of MALI flux output file paths. + mapping_file : str + Path to the ESMF mapping/weights file. + ismip7_grid_file : str + Path to the ISMIP7 grid file. + output_path : str + Directory for output files. + metadata : dict + Submission metadata dict. + """ + check_flux_files(flux_files) + + print("Preparing concatenated flux file for remapping.") + tmp_flux_file = 'tmp_flux.nc' + process_flux_vars(flux_files, tmp_flux_file) + + print("Remapping flux file.") + processed_file_flux = 'processed_flux.nc' + check_call(["ncremap", + "-i", tmp_flux_file, + "-o", processed_file_flux, + "-m", mapping_file, + "-P", "mpas"]) + + print("Writing processed and remapped flux fields to ISMIP7 file format.") + generate_output_2d_flux_vars(processed_file_flux, + ismip7_grid_file, + output_path, metadata) + + os.remove(tmp_flux_file) + os.remove(processed_file_flux) From 4f31e72ffc2b819f78e7707cc9395ffbee9b292c Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 13:10:24 -0700 Subject: [PATCH 24/33] Make PEP8 compliant --- .../ismip7_postprocessing/grid_and_mapping.py | 42 +-- .../post_process_mali_to_ismip7.py | 258 ++++++++++++------ .../process_1d_variables_ismip7.py | 163 ++++++++--- .../process_flux_variables_ismip7.py | 33 ++- .../process_state_variables_ismip7.py | 104 ++++--- .../recalculate_missing_2d_state_vars.py | 95 ++++--- .../ismip7_postprocessing/validate.py | 3 +- 7 files changed, 458 insertions(+), 240 deletions(-) mode change 100755 => 100644 landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py diff --git a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py index e2a69f60a..a47ec862d 100644 --- a/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py +++ b/landice/output_processing_li/ismip7_postprocessing/grid_and_mapping.py @@ -2,7 +2,6 @@ from subprocess import check_call import os import netCDF4 -import xarray as xr VALID_EXPERIMENTS = [f"C{i:03d}" for i in range(1, 12)] @@ -56,11 +55,11 @@ def check_exp_name(exp): f"Valid experiments are: {', '.join(VALID_EXPERIMENTS)}") print(f"Experiment name '{exp}' is valid.") + def check_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): """ Ensure the ISMIP7 grid file has 'x' and 'y' coordinate variables and that - its corners match the ISMIP7-required extents. If 'x'/'y' are absent a - temporary copy with those variables added is created. + its corners match the ISMIP7-required extents. Parameters ---------- @@ -112,6 +111,7 @@ def check_ismip7_grid_file(ismip7_grid_file_path, res_ismip7_grid): f"upper right corner values at {x[-1]}m and {y[-1]}m") check_ds.close() + def build_mapping_file(mali_mesh_file, mapping_file, res_ismip7_grid, ismip7_grid_file=None, @@ -141,7 +141,7 @@ def build_mapping_file(mali_mesh_file, """ if os.path.exists(mapping_file): - print(f"Mapping file exists. Not building a new one.") + print("Mapping file exists. Not building a new one.") return if ismip7_grid_file is None: @@ -158,10 +158,10 @@ def build_mapping_file(mali_mesh_file, # create the ismip7 scripfile if mapping file does not exist # this is the projection of ismip7 data for Antarctica - print(f"Mapping file does not exist. Building one based on " - f"the input/ouptut meshes") - print(f"Creating temporary scripfiles " - f"for ismip7 grid and mali mesh...") + print("Mapping file does not exist. Building one based on " + "the input/ouptut meshes") + print("Creating temporary scripfiles " + "for ismip7 grid and mali mesh...") # create a scripfile for ismip7 grid args = ["create_scrip_file_from_planar_rectangular_grid", @@ -189,25 +189,25 @@ def build_mapping_file(mali_mesh_file, hostname = os.uname()[1] if hostname.startswith('nid'): args = (["srun", "-n", "12", "ESMF_RegridWeightGen", - "-s", mali_scripfile, - "-d", ismip7_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"]) + "-s", mali_scripfile, + "-d", ismip7_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) else: args = (["ESMF_RegridWeightGen", - "-s", mali_scripfile, - "-d", ismip7_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"]) + "-s", mali_scripfile, + "-d", ismip7_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) print(f"Running remapping command: {' '.join(args)}") check_call(args) # remove the temporary scripfiles once the mapping file is generated - print(f"Removing the temporary mesh and scripfiles...") + print("Removing the temporary mesh and scripfiles...") os.remove(ismip7_scripfile) os.remove(mali_scripfile) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py old mode 100755 new mode 100644 index b5a031aa2..3e5ea4ebb --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -20,74 +20,144 @@ """ import argparse -from subprocess import check_call -import os import glob +import os from datetime import datetime -from grid_and_mapping import build_mapping_file, check_ismip7_grid_file, \ - check_exp_name, check_res -from process_state_variables_ismip7 import process_state_pipeline -from process_1d_variables_ismip7 import generate_output_1d_vars, \ - check_global_stats_files + +from grid_and_mapping import ( + build_mapping_file, + check_exp_name, + check_ismip7_grid_file, + check_res, +) +from process_1d_variables_ismip7 import ( + check_global_stats_files, + generate_output_1d_vars, +) from process_flux_variables_ismip7 import process_flux_pipeline +from process_state_variables_ismip7 import process_state_pipeline DEFAULT_AUTHORS = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' DEFAULT_GROUP = 'Los Alamos National Laboratory, Department of Energy' DEFAULT_MODEL = 'MALI (MPAS-Albany Land Ice model)' DEFAULT_GROUP_NICKNAME = 'DOE' + def main(): - parser = argparse.ArgumentParser( - description=__doc__) - parser.add_argument("-e", "--exp_name", dest="exp", - required=True, - help="ISMIP7 experiment name (e.g., exp05") - parser.add_argument("-s", "--input_state_pattern", dest="input_state_pattern", - required=False, - help="glob pattern matching one or more MALI state output files. " - "Note: wildcard paths should be quoted to avoid shell expansion.") - parser.add_argument("-f", "--input_flux_pattern", dest="input_flux_pattern", - required=False, - help="glob pattern matching one or more MALI flux output files. " - "Note: wildcard paths should be quoted to avoid shell expansion.") - parser.add_argument("-m", "--input_mesh", dest="input_file_mesh", - required=False, help="MALI file with mesh information") - parser.add_argument("-g", "--global_stats_pattern", dest="global_stats_pattern", - required=False, - help="glob pattern matching one or more globalStats.nc " - "files (e.g. 'globalStats_*.nc')." - "Note: wildcard paths should be quoted to avoid shell expansion.") - parser.add_argument("-o", "--output_path", dest="output_path", - required=True, - help="path to which the final output files" - " will be saved") - parser.add_argument("--reuse_mapping_file", dest="reuse_mapping_file", - required=False, - help="existing mapping file name to reuse") - parser.add_argument("--ismip7_grid_file", dest="ismip7_grid_file", - help="Input ismip7 mesh file.") - parser.add_argument("--method", dest="method_remap", default="conserve", - required=False, - help="mapping method. Default='conserve'") - parser.add_argument("--res", dest="res_ismip7_grid", - required=True, - help="resolution of the ismip7 grid, in kilometers: 16, 8, 4, 2, 1") - parser.add_argument("--icesheet", dest="icesheet", - required=True, - choices=['AIS', 'GIS'], - help="ice sheet domain: 'AIS' (Antarctica) or 'GIS' (Greenland)") - parser.add_argument("--authors", dest="authors", - required=False, default=DEFAULT_AUTHORS, - help=f"author string for output file metadata " - f"(default: '{DEFAULT_AUTHORS}')") - parser.add_argument("--group", dest="group", - required=False, default=DEFAULT_GROUP, - help=f"group/institution string for output file metadata " - f"(default: '{DEFAULT_GROUP}')") - parser.add_argument("--group_nickname", dest="group_nickname", - required=False, default=DEFAULT_GROUP_NICKNAME, - help=f"short group nickname used in output filenames " - f"(default: '{DEFAULT_GROUP_NICKNAME}')") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "-e", + "--exp_name", + dest="exp", + required=True, + help="ISMIP7 experiment name (e.g., exp05)", + ) + parser.add_argument( + "-s", + "--input_state_pattern", + dest="input_state_pattern", + required=False, + help=( + "glob pattern matching one or more MALI state output files. " + "Note: wildcard paths should be quoted to avoid shell expansion." + ), + ) + parser.add_argument( + "-f", + "--input_flux_pattern", + dest="input_flux_pattern", + required=False, + help=( + "glob pattern matching one or more MALI flux output files. " + "Note: wildcard paths should be quoted to avoid shell expansion." + ), + ) + parser.add_argument( + "-m", + "--input_mesh", + dest="input_file_mesh", + required=False, + help="MALI file with mesh information", + ) + parser.add_argument( + "-g", + "--global_stats_pattern", + dest="global_stats_pattern", + required=False, + help=( + "glob pattern matching one or more globalStats.nc files " + "(e.g. 'globalStats_*.nc'). " + "Note: wildcard paths should be quoted to avoid shell expansion." + ), + ) + parser.add_argument( + "-o", + "--output_path", + dest="output_path", + required=True, + help="path to which the final output files will be saved", + ) + parser.add_argument( + "--reuse_mapping_file", + dest="reuse_mapping_file", + required=False, + help="existing mapping file name to reuse", + ) + parser.add_argument( + "--ismip7_grid_file", + dest="ismip7_grid_file", + help="Input ismip7 mesh file.", + ) + parser.add_argument( + "--method", + dest="method_remap", + default="conserve", + required=False, + help="mapping method. Default='conserve'", + ) + parser.add_argument( + "--res", + dest="res_ismip7_grid", + required=True, + help="resolution of the ismip7 grid, in kilometers: 16, 8, 4, 2, 1", + ) + parser.add_argument( + "--icesheet", + dest="icesheet", + required=True, + choices=['AIS', 'GIS'], + help="ice sheet domain: 'AIS' (Antarctica) or 'GIS' (Greenland)", + ) + parser.add_argument( + "--authors", + dest="authors", + required=False, + default=DEFAULT_AUTHORS, + help=( + "author string for output file metadata " + f"(default: '{DEFAULT_AUTHORS}')" + ), + ) + parser.add_argument( + "--group", + dest="group", + required=False, + default=DEFAULT_GROUP, + help=( + "group/institution string for output file metadata " + f"(default: '{DEFAULT_GROUP}')" + ), + ) + parser.add_argument( + "--group_nickname", + dest="group_nickname", + required=False, + default=DEFAULT_GROUP_NICKNAME, + help=( + "short group nickname used in output filenames " + f"(default: '{DEFAULT_GROUP_NICKNAME}')" + ), + ) args = parser.parse_args() check_exp_name(args.exp) @@ -103,48 +173,54 @@ def main(): 'date': datetime.now().strftime("%d-%b-%Y"), } - print("\n---Processing remapping file---") # Only do remapping steps if we have 2d files to process - if not args.input_state_pattern is None or not args.input_flux_pattern is None: - # Check grid file and res if 2d variables are to be processed + if ( + args.input_state_pattern is not None or + args.input_flux_pattern is not None): check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) - # Check mapping method and either reuse an existing map or create a new one. - # Note: the function 'building_mapping_file' requires the mpas mesh tool - # script 'create_SCRIP_file_from_planar_rectangular_grid.py' + method_remap = args.method_remap if args.reuse_mapping_file is not None: if not os.path.exists(args.reuse_mapping_file): - raise FileNotFoundError(f"Mapping file to reuse not found: " - f"{args.reuse_mapping_file}") + raise FileNotFoundError( + "Mapping file to reuse not found: " + f"{args.reuse_mapping_file}" + ) print(f"Reusing existing mapping file: {args.reuse_mapping_file}") mapping_file = args.reuse_mapping_file else: if args.input_file_mesh is None: - raise ValueError("--input_mesh is required when creating a new " - "mapping file.") + raise ValueError( + "--input_mesh is required when creating " + "a new mapping file." + ) created_at = datetime.now().strftime("%Y%m%dT%H%M%S") - mapping_file = f"mapping_mali_to_ismip7.{method_remap}.{created_at}.nc" + mapping_file = ( + f"mapping_mali_to_ismip7.{method_remap}.{created_at}.nc" + ) - print(f"Creating new mapping file." + print("Creating new mapping file. " f"Mapping method used: {method_remap}") - - build_mapping_file(args.input_file_mesh, mapping_file, - args.res_ismip7_grid, args.ismip7_grid_file, - method_remap) + build_mapping_file( + args.input_file_mesh, + mapping_file, + args.res_ismip7_grid, + args.ismip7_grid_file, + method_remap, + ) print("---Processing remapping file complete---\n") - # define the path to which the output (processed) files will be saved output_path = args.output_path print(f"Using output path: {output_path}") if not os.path.isdir(output_path): os.makedirs(output_path) - # process 1D variables if args.global_stats_pattern is None: - print("--- No global stats pattern provided; skipping 1D variable processing.") + print("--- No global stats pattern provided; " + "skipping 1D variable processing.") else: print("\n---Processing global stats file(s)---") global_stats_files = sorted(glob.glob(args.global_stats_pattern)) @@ -152,26 +228,38 @@ def main(): generate_output_1d_vars(global_stats_files, output_path, metadata) print("---Processing global stats file(s) complete---\n") - # process 2d state variables if args.input_state_pattern is None: - print("--- MALI state pattern is not provided, thus it will not be processed.") + print("--- MALI state pattern is not provided, " + "thus it will not be processed.") else: print("\n---Processing state file(s)---") state_files = sorted(glob.glob(args.input_state_pattern)) - process_state_pipeline(state_files, mapping_file, args.ismip7_grid_file, - output_path, metadata) + process_state_pipeline( + state_files, + mapping_file, + args.ismip7_grid_file, + output_path, + metadata, + ) print("---Processing state file(s) complete---\n") - # process 2d flux variables if args.input_flux_pattern is None: - print("--- MALI flux pattern is not provided, thus it will not be processed.") + print("--- MALI flux pattern is not provided, " + "thus it will not be processed.") else: print("\n---Processing flux file(s)---") flux_files = sorted(glob.glob(args.input_flux_pattern)) - process_flux_pipeline(flux_files, mapping_file, args.ismip7_grid_file, - output_path, metadata) + process_flux_pipeline( + flux_files, + mapping_file, + args.ismip7_grid_file, + output_path, + metadata, + ) print("---Processing flux file(s) complete---\n") + print("---All processing complete---") + if __name__ == "__main__": main() diff --git a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py index 0a769670d..97934fb81 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_1d_variables_ismip7.py @@ -4,11 +4,9 @@ """ from netCDF4 import Dataset -from datetime import date import numpy as np import os import sys -import glob import xarray as xr from validate import validate_mali_files @@ -32,12 +30,28 @@ def check_global_stats_files(files): validate_mali_files(files, EXPECTED_VARIABLES, label='globalStats') -def _write_state_var(varname, data_values, time_days, standard_name, units, - variable_desc, output_path, simulationStartDate, metadata): - """Write a single 1D state (snapshot) variable to a NETCDF4_CLASSIC file.""" +def _build_output_filename(output_path, varname, metadata): + """Build a standard ISMIP7 output filename for a given variable.""" + return ( + f'{output_path}/{varname}_{metadata["icesheet"]}_' + f'{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc' + ) + + +def _write_state_var( + varname, + data_values, + time_days, + standard_name, + units, + variable_desc, + output_path, + simulationStartDate, + metadata): + """Write one 1D state (snapshot) variable to NETCDF4_CLASSIC.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', - 'w', format='NETCDF4_CLASSIC') + filename = _build_output_filename(output_path, varname, metadata) + ds_out = Dataset(filename, 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) var_out = ds_out.createVariable(varname, 'd', ('time',)) time_out = ds_out.createVariable('time', 'd', ('time',)) @@ -60,10 +74,10 @@ def _write_state_var(varname, data_values, time_days, standard_name, units, def _write_flux_var(varname, data_values, days_min, days_max, standard_name, units, variable_desc, output_path, simulationStartDate, metadata): - """Write a single 1D flux (time-averaged) variable to a NETCDF4_CLASSIC file.""" + """Write one 1D flux (time-averaged) variable to NETCDF4_CLASSIC.""" nt = len(data_values) - ds_out = Dataset(f'{output_path}/{varname}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', - 'w', format='NETCDF4_CLASSIC') + filename = _build_output_filename(output_path, varname, metadata) + ds_out = Dataset(filename, 'w', format='NETCDF4_CLASSIC') ds_out.createDimension('time', nt) ds_out.createDimension('bnds', 2) var_out = ds_out.createVariable(varname, 'd', ('time',)) @@ -98,7 +112,8 @@ def generate_output_1d_vars(files, output_path, metadata): output_path : str Directory for output files. metadata : dict - Submission metadata with keys: exp, icesheet, authors, group, model, date. + Submission metadata with keys: + exp, icesheet, authors, group, model, date. """ if output_path is None or not os.path.exists(output_path): @@ -108,14 +123,19 @@ def generate_output_1d_vars(files, output_path, metadata): decode_cf=False, data_vars='minimal', coords='minimal', compat='override') with xr.open_dataset(files[0], decode_cf=False) as ds_first: - simulationStartTime = (ds_first['simulationStartTime'].values - .tobytes().decode('utf-8').strip().strip('\x00')) + simulationStartTime = ( + ds_first['simulationStartTime'] + .values.tobytes().decode('utf-8').strip().strip('\x00') + ) daysSinceStart = ds['daysSinceStart'].values dt = ds['deltat'].values simulationStartDate = simulationStartTime.split("_")[0] if simulationStartDate[5:10] != '01-01': ds.close() - sys.exit("Error: simulationStartTime for globalStats file is not on Jan. 1.") + sys.exit( + "Error: simulationStartTime for globalStats file " + "is not on Jan. 1." + ) refYear = int(simulationStartDate[0:4]) decYears = refYear + daysSinceStart / 365.0 endYr = decYears[-1] @@ -130,13 +150,17 @@ def generate_output_1d_vars(files, output_path, metadata): # Flux fields should never use the Jan. 1 time level at the start of the # year as part of the averaging. # For year conventions here, for state fields, the year is the snapshot at - # the start of the year, e.g., state year 2000 means the snapshot at Jan. 1, 2000. + # the start of the year, e.g., state year 2000 means the snapshot at + # Jan. 1, 2000. # For flux fields, the years is the calendar year being averaged over, - # e.g., flux year 2000 is the average between Jan. 1, 2000, and Jan. 1, 2001. - # Note this year convention differs from the first column in table in A2.3.2 at + # e.g., flux year 2000 is the average between Jan. 1, 2000, + # and Jan. 1, 2001. + # Note this year convention differs from the first column in table in + # A2.3.2 at # https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP7-Projections2300-Antarctica#A2.3.3_Table_A1:_Variable_request_for_ISMIP6 # but that year indexing convention ultimately doesn't matter because the - # time coordinates in these files uses units of days since a reference date, + # time coordinates in these files uses units of days since a + # reference date, # and it does not use a year indexing convention at all. if decYears[0] == np.round(decYears[0]): # The initial time level will only be on an even year (Jan. 1) @@ -144,21 +168,31 @@ def generate_output_1d_vars(files, output_path, metadata): # even year in the state processing. We also want the state snapshot # at the final (even) year in the output. # The flux processing should start with the first year, which covers a - # full 12 months. We exclude the final year, which is just a Jan. 1 posting. + # full 12 months. We exclude the final year, which is just a Jan. 1 + # posting. years_state = np.arange(decYears[0], endYr + 1) years_flux = np.arange(decYears[0], endYr) else: - # For projection runs, the first state snapshot we want is the first Jan. 1, + # For projection runs, the first state snapshot we want is the + # first Jan. 1, # which we be the first even year after the initial time in the file. - # For flux files, the first full year we want to process is the year of the - # first time level in the file. As with hist, we exclude the final year, + # For flux files, the first full year we want to process is the + # year of the + # first time level in the file. As with hist, we exclude the + # final year, # which is just a Jan. 1 posting. years_state = np.arange(np.ceil(decYears[0]), endYr + 1) years_flux = np.arange(np.floor(decYears[0]), endYr) nt_state = len(years_state) nt_flux = len(years_flux) - print(f'For state processing, using start year={years_state[0]} and end year={years_state[-1]}.') - print(f'For flux processing, using start year={years_flux[0]} and end year={years_flux[-1]}.') + print( + "For state processing, using start " + f"year={years_state[0]} and end year={years_state[-1]}." + ) + print( + "For flux processing, using start " + f"year={years_flux[0]} and end year={years_flux[-1]}." + ) # read in state variables vol = ds['totalIceVolume'].values @@ -172,21 +206,29 @@ def generate_output_1d_vars(files, output_path, metadata): # clean out some garbage values we can't account for ind = np.nonzero(bmbGr > 1.0e18)[0] if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal>1.0e18") + print( + f"WARNING: Found { + len(ind)} values of totalGroundedBasalMassBal>1.0e18") bmbGr[ind] = np.nan ind = np.nonzero(bmbGr < -1.0e18)[0] if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalGroundedBasalMassBal<-1.0e18") + print( + f"WARNING: Found { + len(ind)} values of totalGroundedBasalMassBal<-1.0e18") bmbGr[ind] = np.nan bmbFlt = ds['totalFloatingBasalMassBal'].values.copy() # clean out some garbage values we can't account for - ind = np.nonzero(bmbFlt>1.0e18)[0] + ind = np.nonzero(bmbFlt > 1.0e18)[0] if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal>1.0e18") + print( + f"WARNING: Found { + len(ind)} values of totalFloatingBasalMassBal>1.0e18") bmbFlt[ind] = np.nan - ind = np.nonzero(bmbFlt<-1.0e18)[0] + ind = np.nonzero(bmbFlt < -1.0e18)[0] if len(ind) > 0: - print(f"WARNING: Found {len(ind)} values of totalFloatingBasalMassBal<-1.0e18") + print( + f"WARNING: Found { + len(ind)} values of totalFloatingBasalMassBal<-1.0e18") bmbFlt[ind] = np.nan cfx = ds['totalCalvingFlux'].values fmfx = ds['totalFaceMeltingFlux'].values @@ -213,7 +255,9 @@ def generate_output_1d_vars(files, output_path, metadata): # Use isclose to avoid floating-point equality issues. ind_snap = np.where(np.isclose(decYears, years_state[i]))[0] if len(ind_snap) == 0: - raise ValueError(f"No state snapshot found for year {years_state[i]}.") + raise ValueError( + f"No state snapshot found for year { + years_state[i]}.") if len(ind_snap) > 1: print(f"WARNING: Found {len(ind_snap)} snapshots for year " f"{years_state[i]}; using the first one.") @@ -230,8 +274,12 @@ def generate_output_1d_vars(files, output_path, metadata): # this is for the flux variables for i in range(nt_flux): - ind_avg = np.where(np.logical_and(decYears > years_flux[i], - decYears <= (years_flux[i] + 1.0)))[0] + ind_avg = np.where( + np.logical_and( + decYears > years_flux[i], + decYears <= ( + years_flux[i] + + 1.0)))[0] if len(ind_avg) == 0: raise ValueError(f"No flux averaging samples found for year " f"{years_flux[i]}.") @@ -241,7 +289,7 @@ def generate_output_1d_vars(files, output_path, metadata): cfxi = cfx[ind_avg] fmfxi = fmfx[ind_avg] gfxi = gfx[ind_avg] - dti = dt[ind_avg] + dti = dt[ind_avg] # take the average of the flux variables smb_avg[i] = np.nansum(smbi * dti) / np.nansum(dti) @@ -269,27 +317,52 @@ def generate_output_1d_vars(files, output_path, metadata): _write_state_var('limnsw', vaf_snapshot * 910, days_snapshot, 'land_ice_mass_not_displacing_sea_water', 'kg', 'Mass above floatation', **common) - _write_state_var('iareagr', gia_snapshot, days_snapshot, - 'grounded_ice_sheet_area', 'm2', 'Grounded ice area', **common) - _write_state_var('iareafl', fia_snapshot, days_snapshot, - 'floating_ice_shelf_area', 'm2', 'Floating ice area', **common) + _write_state_var( + 'iareagr', + gia_snapshot, + days_snapshot, + 'grounded_ice_sheet_area', + 'm2', + 'Grounded ice area', + **common) + _write_state_var( + 'iareafl', + fia_snapshot, + days_snapshot, + 'floating_ice_shelf_area', + 'm2', + 'Floating ice area', + **common) # --- flux (time-averaged) variables --- _write_flux_var('tendacabf', smb_avg / 31536000.0, days_min, days_max, 'tendency_of_land_ice_mass_due_to_surface_mass_balance', 'kg s-1', 'Total SMB flux', **common) - _write_flux_var('tendlibmassbfgr', bmbGr_avg / 31536000.0, days_min, days_max, - 'tendency_of_land_ice_mass_due_to_basal_mass_balance', - 'kg s-1', 'Total BMB flux beneath grounded ice', **common) - _write_flux_var('tendlibmassbffl', bmbFlt_avg / 31536000.0, days_min, days_max, - 'tendency_of_land_ice_mass_due_to_basal_mass_balance', - 'kg s-1', 'Total BMB flux beneath floating ice', **common) + _write_flux_var( + 'tendlibmassbfgr', + bmbGr_avg / 31536000.0, + days_min, + days_max, + 'tendency_of_land_ice_mass_due_to_basal_mass_balance', + 'kg s-1', + 'Total BMB flux beneath grounded ice', + **common) + _write_flux_var( + 'tendlibmassbffl', + bmbFlt_avg / 31536000.0, + days_min, + days_max, + 'tendency_of_land_ice_mass_due_to_basal_mass_balance', + 'kg s-1', + 'Total BMB flux beneath floating ice', + **common) # tendlicalvf: sign convention — calving removes mass, so negate _write_flux_var('tendlicalvf', -cfx_avg / 31536000.0, days_min, days_max, 'tendency_of_land_ice_mass_due_to_calving', 'kg s-1', 'Total calving flux', **common) # tendlifmassbf: in ISMIP7 this is ice-front melting only (not calving) - _write_flux_var('tendlifmassbf', -fmfx_avg / 31536000.0, days_min, days_max, + _write_flux_var('tendlifmassbf', -fmfx_avg / 31536000.0, days_min, + days_max, 'tendency_of_land_ice_mass_due_to_ice_front_melting', 'kg s-1', 'Total ice front melting flux', **common) _write_flux_var('tendligroundf', gfx_avg / 31536000.0, days_min, days_max, diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index 448b82b9a..d3cba4af0 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -6,10 +6,8 @@ from netCDF4 import Dataset import xarray as xr import numpy as np -from datetime import date from subprocess import check_call -import os, sys -import warnings +import os from validate import validate_mali_files @@ -48,7 +46,9 @@ def process_flux_vars(files, tmp_file): data_vars='minimal', coords='minimal', compat='override') - if 'daysSinceStart' in ds_flux and 'units' in ds_flux['daysSinceStart'].attrs: + if ( + 'daysSinceStart' in ds_flux and + 'units' in ds_flux['daysSinceStart'].attrs): del ds_flux['daysSinceStart'].attrs['units'] ds_flux.to_netcdf(tmp_file) ds_flux.close() @@ -57,7 +57,6 @@ def process_flux_vars(files, tmp_file): def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, var_units, var_varname, remapped_mali_flux_file, ismip7_grid_file, output_path, metadata): - """ mali_var_name: variable name on MALI side ismip7_var_name: variable name required by ISMIP7 @@ -77,34 +76,37 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, data = Dataset(remapped_mali_flux_file, 'r') data.set_auto_mask(False) iceMask = data.variables['iceMask'][:, :, :] - simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + simulationStartTime = data.variables['simulationStartTime'][:].tostring( + ).decode('utf-8').strip().strip('\x00') simulationStartDate = simulationStartTime.split("_")[0] timeBndsMin = data.variables['timeBndsMin'][:] timeBndsMax = data.variables['timeBndsMax'][:] - if not mali_var_name in data.variables: + if mali_var_name not in data.variables: print(f"WARNING: {mali_var_name} not present. Skipping.") data.close() return - var_mali = data.variables[mali_var_name][:,:,:] + var_mali = data.variables[mali_var_name][:, :, :] var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', - 'w', format='NETCDF4_CLASSIC') + dataOut = Dataset( + f'{output_path}/{ismip7_var_name}_{ + metadata["icesheet"]}_{ + metadata["group_nickname"]}_MALI_{ + metadata["exp"]}.nc', + 'w', + format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('bnds', 2) timebndsValues = dataOut.createVariable('time_bnds', 'd', ('time', 'bnds')) dataOut.createDimension('x', lonN) dataOut.createDimension('y', latN) dataValues = dataOut.createVariable(ismip7_var_name, 'd', - ('time', 'y', 'x'), fill_value=np.NAN) + ('time', 'y', 'x'), fill_value=np.NAN) xValues = dataOut.createVariable('x', 'd', ('x')) yValues = dataOut.createVariable('y', 'd', ('y')) timeValues = dataOut.createVariable('time', 'd', ('time')) - AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' - DATE_STR = metadata['date'] - for i in range(timeSteps): mask = iceMask[i, :, :] tmp = var_mali[i, :, :] @@ -216,7 +218,8 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, def process_flux_pipeline(flux_files, mapping_file, ismip7_grid_file, output_path, metadata): """ - Full flux-variable processing pipeline: validate, concatenate, remap, write. + Full flux-variable processing pipeline: + validate, concatenate, remap, write. Parameters ---------- diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 6896641ce..0321ca13b 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -6,8 +6,7 @@ from netCDF4 import Dataset import xarray as xr import numpy as np -from datetime import date -import os, sys +import os from subprocess import check_call from validate import validate_mali_files @@ -27,6 +26,7 @@ def check_state_files(files): """ validate_mali_files(files, EXPECTED_STATE_VARIABLES, label='state') + def process_state_vars(files, tmp_file): """ files: list of MALI state output file paths @@ -46,16 +46,16 @@ def process_state_vars(files, tmp_file): del inputfile_state_vars['daysSinceStart'].attrs['units'] # get the mesh description data - nCells = inputfile_state_vars.dims['nCells'] - nTime = inputfile_state_vars.dims['Time'] nLayer = inputfile_state_vars.dims['nVertLevels'] nInterface = nLayer + 1 # inputfile_state_vars.dims['nVertInterfaces'] cellMask = inputfile_state_vars['cellMask'][:, :] basalTemperature = inputfile_state_vars['basalTemperature'][:, :] betaSolve = inputfile_state_vars['betaSolve'][:, :] - inputfile_state_vars['litempbotfl'] = basalTemperature * (cellMask[:, :] & 4) / 4 - inputfile_state_vars['litempbotgr'] = basalTemperature * (1 - (cellMask[:, :] & 4) / 4) + inputfile_state_vars['litempbotfl'] = basalTemperature * \ + (cellMask[:, :] & 4) / 4 + inputfile_state_vars['litempbotgr'] = basalTemperature * \ + (1 - (cellMask[:, :] & 4) / 4) uxsurf = inputfile_state_vars['uReconstructX'][:, :, 0] uysurf = inputfile_state_vars['uReconstructY'][:, :, 0] @@ -66,22 +66,41 @@ def process_state_vars(files, tmp_file): inputfile_state_vars['uReconstructX_base'] = uxbase inputfile_state_vars['uReconstructY_base'] = uybase - inputfile_state_vars['upperSurface'] = np.maximum(0.0, inputfile_state_vars['upperSurface']) + inputfile_state_vars['upperSurface'] = np.maximum( + 0.0, inputfile_state_vars['upperSurface']) + + floating_mask = (cellMask[:, :] & 4) / 4 + dynamic_mask = (cellMask[:, :] & 2) / 2 + grounded_mask = (cellMask[:, :] * 0 + 1) - floating_mask - inputfile_state_vars['sftflf'] = (cellMask[:, :] & 4) / 4 * (cellMask[:, :] & 2) / 2 # floating and dynamic - inputfile_state_vars['sftgrf'] = ((cellMask[:, :] * 0 + 1) - (cellMask[:, :] & 4) / 4) * (cellMask[:, :] & 2) / 2 # grounded: not-floating & dynamic - inputfile_state_vars['sftgif'] = (cellMask[:, :] & 2) / 2 # grounded: dynamic ice - inputfile_state_vars['strbasemag'] = betaSolve[:, :] * ((uxbase[:, :]) ** 2 + (uybase[:, :]) ** 2) **0.5 \ - * (3600.0 * 24.0 * 365.0) \ - * (cellMask[:, :] * 0 + 1 - (cellMask[:, :] & 4) / 4) * (cellMask[:, :] & 2) / 2 + # floating and dynamic + inputfile_state_vars['sftflf'] = floating_mask * dynamic_mask + # grounded: not-floating and dynamic + inputfile_state_vars['sftgrf'] = grounded_mask * dynamic_mask + # dynamic ice + inputfile_state_vars['sftgif'] = dynamic_mask + + speed_base = (uxbase[:, :] ** 2 + uybase[:, :] ** 2) ** 0.5 + seconds_per_year = 3600.0 * 24.0 * 365.0 + inputfile_state_vars['strbasemag'] = ( + betaSolve[:, :] * speed_base * seconds_per_year * grounded_mask * + dynamic_mask + ) inputfile_state_vars.to_netcdf(tmp_file) inputfile_state_vars.close() -def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, - var_units, var_varname, remapped_mali_outputfile, - ismip7_grid_file, output_path, metadata): +def write_netcdf_2d_state_vars( + mali_var_name, + ismip7_var_name, + var_std_name, + var_units, + var_varname, + remapped_mali_outputfile, + ismip7_grid_file, + output_path, + metadata): """ mali_var_name: variable name on MALI side ismip7_var_name: variable name required by ISMIP7 @@ -100,30 +119,33 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, data = Dataset(remapped_mali_outputfile, 'r') data.set_auto_mask(False) - simulationStartTime = data.variables['simulationStartTime'][:].tostring().decode('utf-8').strip().strip('\x00') + simulationStartTime = data.variables['simulationStartTime'][:].tostring( + ).decode('utf-8').strip().strip('\x00') simulationStartDate = simulationStartTime.split("_")[0] daysSinceStart = data.variables['daysSinceStart'][:] var_sftgif = data.variables['sftgif'][:, :, :] var_sftgrf = data.variables['sftgrf'][:, :, :] var_sftflf = data.variables['sftflf'][:, :, :] - var_mali = data.variables[mali_var_name][:,:,:] + var_mali = data.variables[mali_var_name][:, :, :] var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN timeSteps, latN, lonN = np.shape(var_mali) - dataOut = Dataset(f'{output_path}/{ismip7_var_name}_{metadata["icesheet"]}_{metadata["group_nickname"]}_MALI_{metadata["exp"]}.nc', - 'w', format='NETCDF4_CLASSIC') + dataOut = Dataset( + f'{output_path}/{ismip7_var_name}_{ + metadata["icesheet"]}_{ + metadata["group_nickname"]}_MALI_{ + metadata["exp"]}.nc', + 'w', + format='NETCDF4_CLASSIC') dataOut.createDimension('time', timeSteps) dataOut.createDimension('x', lonN) dataOut.createDimension('y', latN) dataValues = dataOut.createVariable(ismip7_var_name, 'd', - ('time', 'y', 'x'), fill_value=np.NAN) + ('time', 'y', 'x'), fill_value=np.NAN) xValues = dataOut.createVariable('x', 'd', ('x')) yValues = dataOut.createVariable('y', 'd', ('y')) timeValues = dataOut.createVariable('time', 'd', ('time')) timeValues[:] = daysSinceStart - AUTHOR_STR = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' - DATE_STR = metadata['date'] - for i in range(timeSteps): if ismip7_var_name == 'sftgif': dataValues[i, :, :] = var_mali[i, :, :] @@ -167,7 +189,7 @@ def write_netcdf_2d_state_vars(mali_var_name, ismip7_var_name, var_std_name, def process_state_pipeline(state_files, mapping_file, ismip7_grid_file, - output_path, metadata): + output_path, metadata): """ Full state-variable processing pipeline: validate, adjust, remap, write. @@ -216,34 +238,40 @@ def generate_output_2d_state_vars(file_remapped_mali_state, output_path: path to which the final output files are saved """ - # ----------- lithk ------------------ - write_netcdf_2d_state_vars('thickness','lithk', 'land_ice_thickness', + write_netcdf_2d_state_vars('thickness', 'lithk', 'land_ice_thickness', 'm', 'Ice thickness', file_remapped_mali_state, ismip7_grid_file, output_path, metadata) # ----------- orog ------------------ - write_netcdf_2d_state_vars('upperSurface','orog', 'surface_altitude', 'm', + write_netcdf_2d_state_vars('upperSurface', 'orog', 'surface_altitude', 'm', 'Surface elevation', file_remapped_mali_state, ismip7_grid_file, output_path, metadata) # ----------- base ------------------ - write_netcdf_2d_state_vars('lowerSurface','base', 'base_altitude', 'm', + write_netcdf_2d_state_vars('lowerSurface', 'base', 'base_altitude', 'm', 'Base elevation', file_remapped_mali_state, ismip7_grid_file, output_path, metadata) # ----------- topg ------------------ - write_netcdf_2d_state_vars('bedTopography','topg', 'bedrock_altitude', 'm', - 'Bedrock elevation', - file_remapped_mali_state, - ismip7_grid_file, output_path, metadata) + write_netcdf_2d_state_vars( + 'bedTopography', + 'topg', + 'bedrock_altitude', + 'm', + 'Bedrock elevation', + file_remapped_mali_state, + ismip7_grid_file, + output_path, + metadata) # ----------- hfgeoubed------------------ # Note: even though this is a flux variable, we are taking a snapshot of it - # as it does not change with time #Uncomment the function all once basalHeatFlux is outputted in the output stream + # as it does not change with time. + # Uncomment once basalHeatFlux is outputted in the output stream. # write_netcdf_2d_state_vars('basalHeatFlux', 'hfgeoubed', # 'upward_geothermal_heat_flux_in_land_ice', # 'W m-2', 'Geothermal heat flux', @@ -279,8 +307,8 @@ def generate_output_2d_state_vars(file_remapped_mali_state, ismip7_grid_file, output_path, metadata) # ----------- zvelsurf & zvelbase ------------------ - # ISMIP7 requires these variables, but MALI does not output them. - # So, we are not processing/writing these variables out. + # ISMIP7 requires these variables, but MALI does not output them. + # So, we are not processing/writing these variables out. # ----------- xvelmean ------------------ write_netcdf_2d_state_vars('xvelmean', 'xvelmean', @@ -325,7 +353,7 @@ def generate_output_2d_state_vars(file_remapped_mali_state, ismip7_grid_file, output_path, metadata) # ----------- sftgif ------------------ - write_netcdf_2d_state_vars('sftgif','sftgif', + write_netcdf_2d_state_vars('sftgif', 'sftgif', 'land_ice_area_fraction', '1', 'Land ice area fraction', file_remapped_mali_state, @@ -339,7 +367,7 @@ def generate_output_2d_state_vars(file_remapped_mali_state, ismip7_grid_file, output_path, metadata) # ----------- sftflf ------------------ - write_netcdf_2d_state_vars('sftflf','sftflf', + write_netcdf_2d_state_vars('sftflf', 'sftflf', 'floating_ice_shelf_area_fraction', '1', 'Floating ice shelf area fraction', file_remapped_mali_state, diff --git a/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py b/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py index 315715677..226af4ad6 100755 --- a/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py +++ b/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py @@ -2,21 +2,20 @@ """ This script copies a restart file of a MALI simulation -and re-calculates missing state variables for a +and re-calculates missing state variables for a missing time level and writes them to an updated restart file. """ import argparse import os -import shutil import xarray as xr import numpy as np def main(): parser = argparse.ArgumentParser( - description='process MALI outputs for the ISMIP7' - 'submission') + description='process MALI outputs for the ISMIP7' + 'submission') parser.add_argument("-f", "--file", dest="file_in", required=True, help="restart file to be read in") @@ -25,33 +24,38 @@ def main(): help="output file name") parser.add_argument("-p", "--output_file_path", dest="output_path") - + args = parser.parse_args() - + # read in a restart file that needs to be re-written if args.file_in is None: print("--- restart file is not provided. Aborting... ---") else: print("\n--- Reading in the restart file ---") - file_in = xr.open_dataset(args.file_in, decode_times=False, decode_cf=False) + file_in = xr.open_dataset( + args.file_in, + decode_times=False, + decode_cf=False) # get needed info from restart file cellMask = file_in['cellMask'][:, :] - thickness = file_in['thickness'][:,:] - bedTopography = file_in['bedTopography'][:,:] - sfcAirTemp = file_in['surfaceAirTemperature'][:,:] - uReconstructX = file_in['uReconstructX'][:,:,:] - uReconstructY = file_in['uReconstructY'][:,:,:] + thickness = file_in['thickness'][:, :] + bedTopography = file_in['bedTopography'][:, :] + sfcAirTemp = file_in['surfaceAirTemperature'][:, :] + uReconstructX = file_in['uReconstructX'][:, :, :] + uReconstructY = file_in['uReconstructY'][:, :, :] layerThicknessFractions = file_in['layerThicknessFractions'] nTime = file_in.dims['Time'] nCells = file_in.dims['nCells'] nVertLevels = file_in.dims['nVertLevels'] - # xtime needs some massaging for xarray not to mangle it + # xtime needs some massaging so xarray does not mangle it xtime = file_in['xtime'] xtimeStr = xtime.data.tobytes().decode() # convert to str - xtime2 = xr.DataArray(np.array([xtimeStr], dtype = np.dtype(('S', 64))), dims = ['Time']) # convert back to char array + xtime2 = xr.DataArray( + np.array([xtimeStr], dtype=np.dtype(('S', 64))), + dims=['Time']) # convert back to char array # followed example here: https://github.com/pydata/xarray/issues/3407 floating_iceMask = (cellMask[:, :] & 4) // 4 @@ -61,41 +65,50 @@ def main(): print(f'nTime={nTime}, nCells={nCells}') - layerInterfaceFractions = np.zeros(nVertLevels+1, dtype=float) + layerInterfaceFractions = np.zeros(nVertLevels + 1, dtype=float) lowerSfc = np.zeros([nTime, nCells], dtype=float) upperSfc = np.zeros([nTime, nCells], dtype=float) sfcTemp = np.zeros([nTime, nCells], dtype=float) xvelmean = np.zeros([nTime, nCells], dtype=float) yvelmean = np.zeros([nTime, nCells], dtype=float) - # the following need to be in the file so ncrcat will work but processing won't use + # Needed in the file so ncrcat will work, but processing will not use # values, so can leave as zeros surfaceSpeed = np.zeros([nTime, nCells], dtype=float) vonMisesStress = np.zeros([nTime, nCells], dtype=float) deltat = np.zeros([nTime,], dtype=float) daysSinceStart = np.zeros([nTime,], dtype=float) - + print("\n--- calculating the missing state variables ---") - # layerInterfaceFractions are the fraction associated with each interface + # layerInterfaceFractions are the fraction associated with each + # interface layerInterfaceFractions[0] = 0.5 * layerThicknessFractions[0] for k in range(1, nVertLevels): - layerInterfaceFractions[k] = 0.5 * (layerThicknessFractions[k-1] - + layerThicknessFractions[k]) - layerInterfaceFractions[nVertLevels] = 0.5 * layerThicknessFractions[nVertLevels-1] + layerInterfaceFractions[k] = 0.5 * (layerThicknessFractions[k - 1] + + layerThicknessFractions[k]) + layerInterfaceFractions[nVertLevels] = 0.5 * \ + layerThicknessFractions[nVertLevels - 1] print("layerThicknessFractions:", layerThicknessFractions[:].data) print("layerInterfaceFractions:", layerInterfaceFractions) for i in range(nTime): # calculate surface temperature (unit in Kelvin) - sfcTemp[i,:] = np.minimum(273.15, sfcAirTemp[i,:]) # 0 celsius = 273 Kelvin + # 0 celsius = 273 Kelvin + sfcTemp[i, :] = np.minimum(273.15, sfcAirTemp[i, :]) print('surfaceTemperature processed') - lowerSfc[i,:] = np.where(floating_iceMask, seaLevel - thickness[i,:] * (rhoi / rhoo), bedTopography[i,:]) - upperSfc[i,:] = lowerSfc[i,:] + thickness[i,:] + lowerSfc[i, :] = np.where( + floating_iceMask, + seaLevel - thickness[i, :] * (rhoi / rhoo), + bedTopography[i, :], + ) + upperSfc[i, :] = lowerSfc[i, :] + thickness[i, :] print('lower/upperSurface processed') - xvelmean[i,:] = np.sum(uReconstructX[i,:,:] * layerInterfaceFractions[:], axis=1) - yvelmean[i,:] = np.sum(uReconstructY[i,:,:] * layerInterfaceFractions[:], axis=1) + xvelmean[i, :] = np.sum( + uReconstructX[i, :, :] * layerInterfaceFractions[:], axis=1) + yvelmean[i, :] = np.sum( + uReconstructY[i, :, :] * layerInterfaceFractions[:], axis=1) print('x/yvelmean processed') # create variable dictionary of fields to include in the new file @@ -109,17 +122,28 @@ def main(): 'yvelmean': (['Time', 'nCells'], yvelmean), 'surfaceSpeed': (['Time', 'nCells'], surfaceSpeed), 'vonMisesStress': (['Time', 'nCells'], vonMisesStress), - 'deltat': (['Time',], deltat ), + 'deltat': (['Time',], deltat), 'daysSinceStart': (['Time',], daysSinceStart), 'xtime': xtime2 - } - dataOut = xr.Dataset(data_vars=out_data_vars) # create xarray dataset object - dataOut.xtime.encoding.update({"char_dim_name": "StrLen"}) # another hacky thing to make xarray handle xtime correctly + } + # create xarray dataset object + dataOut = xr.Dataset(data_vars=out_data_vars) + # another hacky thing to make xarray handle xtime correctly + dataOut.xtime.encoding.update({"char_dim_name": "StrLen"}) # learned this from: https://github.com/pydata/xarray/issues/2895 - print("\n--- copying over unmodified variables from the restart file ---") - for var in ['thickness', 'uReconstructX', 'uReconstructY', 'bedTopography', - 'basalTemperature', 'betaSolve', 'cellMask', 'damage']: + print("\n--- copying over unmodified variables from the restart " + "file ---") + for var in [ + 'thickness', + 'uReconstructX', + 'uReconstructY', + 'bedTopography', + 'basalTemperature', + 'betaSolve', + 'cellMask', + 'damage', + ]: print(" Copying", var) dataOut[var] = file_in[var] @@ -129,7 +153,7 @@ def main(): output_path = os.getcwd() else: output_path = args.output_path - + if not os.path.isdir(output_path): os.makedirs(output_path) @@ -137,8 +161,9 @@ def main(): file_out_path = os.path.join(output_path, args.file_out) dataOut.to_netcdf(file_out_path, mode='w', unlimited_dims=['Time']) file_in.close() - + print("\n--- process complete! ---") + if __name__ == "__main__": main() diff --git a/landice/output_processing_li/ismip7_postprocessing/validate.py b/landice/output_processing_li/ismip7_postprocessing/validate.py index 9dadb1930..f7482852f 100644 --- a/landice/output_processing_li/ismip7_postprocessing/validate.py +++ b/landice/output_processing_li/ismip7_postprocessing/validate.py @@ -16,7 +16,8 @@ def validate_mali_files(files, required_vars, label=''): - Each file contains all required variables - simulationStartTime is consistent across all files - No time overlaps (daysSinceStart) exist between consecutive files - - No unexpectedly large time gaps (> 366 days) exist between consecutive files + - No unexpectedly large time gaps (> 366 days) exist between + consecutive files Parameters ---------- From 41bd1acfdd56283d294eeffc1a20cedb1ed0dd54 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 13:16:29 -0700 Subject: [PATCH 25/33] add usage examples --- .../post_process_mali_to_ismip7.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 3e5ea4ebb..ad2b15d57 100644 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -17,6 +17,30 @@ More info at: https://github.com/ismip/ISM_SimulationChecker/blob/main/conventions/ISMIP7_variable_request.csv https://www.ismip.org/research/ismip7 + +Example usage from testing on initial ISMIP7 submission data: +python post_process_mali_to_ismip7.py \ + -o /pscratch/sd/h/hoffman2/ISMIP7-postprocessing-June30-deadline/AIS/test \ + -g "/pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/output/globalStats_*.nc" \ + --icesheet AIS \ + -e C001 \ + --res 04 \ + -m /pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/relaxed_10yrs_4km.nc \ + -s "/pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/output/output_state_*.nc" \ + --ismip7_grid_file /pscratch/sd/h/hoffman2/ISMIP7-postprocessing-June30-deadline/AIS/misc/af2_AIS_04000m_v1.nc \ + -f "/pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/output/output_state_*.nc" + +And here is an example of only processing 2d state and flux after the mapping file has been created: +python post_process_mali_to_ismip7.py \ + -o /pscratch/sd/h/hoffman2/ISMIP7-postprocessing-June30-deadline/AIS/test \ + --icesheet AIS \ + -e C001 \ + --res 04 \ + -m /pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/relaxed_10yrs_4km.nc \ + -s "/pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/output/output_state_*.nc" \ + -f "/pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/output/output_state_*.nc" \ + --reuse_mapping_file mapping_mali_to_ismip7.conserve.20260626T130744.nc + """ import argparse From 61a15b3be4ddc7207c6cd26c6009caedbb01d036 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 13:22:28 -0700 Subject: [PATCH 26/33] Small error/warning fixes --- .../ismip7_postprocessing/post_process_mali_to_ismip7.py | 6 +++++- .../ismip7_postprocessing/process_flux_variables_ismip7.py | 2 +- .../ismip7_postprocessing/process_state_variables_ismip7.py | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index ad2b15d57..e9a9e5bb8 100644 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -202,7 +202,11 @@ def main(): if ( args.input_state_pattern is not None or args.input_flux_pattern is not None): - check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) + + # If the user has not provided a mapping file to reuse, check that the + # ismip7 grid file is provided and valid + if args.reuse_mapping_file is None: + check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) method_remap = args.method_remap if args.reuse_mapping_file is not None: diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index d3cba4af0..0591cb771 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -76,7 +76,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, data = Dataset(remapped_mali_flux_file, 'r') data.set_auto_mask(False) iceMask = data.variables['iceMask'][:, :, :] - simulationStartTime = data.variables['simulationStartTime'][:].tostring( + simulationStartTime = data.variables['simulationStartTime'][:].tobytes( ).decode('utf-8').strip().strip('\x00') simulationStartDate = simulationStartTime.split("_")[0] timeBndsMin = data.variables['timeBndsMin'][:] diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 0321ca13b..0a474b2bf 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -46,8 +46,8 @@ def process_state_vars(files, tmp_file): del inputfile_state_vars['daysSinceStart'].attrs['units'] # get the mesh description data - nLayer = inputfile_state_vars.dims['nVertLevels'] - nInterface = nLayer + 1 # inputfile_state_vars.dims['nVertInterfaces'] + nLayer = inputfile_state_vars.sizes['nVertLevels'] + nInterface = nLayer + 1 # inputfile_state_vars.sizes['nVertInterfaces'] cellMask = inputfile_state_vars['cellMask'][:, :] basalTemperature = inputfile_state_vars['basalTemperature'][:, :] betaSolve = inputfile_state_vars['betaSolve'][:, :] @@ -119,7 +119,7 @@ def write_netcdf_2d_state_vars( data = Dataset(remapped_mali_outputfile, 'r') data.set_auto_mask(False) - simulationStartTime = data.variables['simulationStartTime'][:].tostring( + simulationStartTime = data.variables['simulationStartTime'][:].tobytes( ).decode('utf-8').strip().strip('\x00') simulationStartDate = simulationStartTime.split("_")[0] daysSinceStart = data.variables['daysSinceStart'][:] From 786fbdbcc60f51a635b0167282392e2c53dbdff1 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 13:23:22 -0700 Subject: [PATCH 27/33] Remove unused file --- .../recalculate_missing_2d_state_vars.py | 169 ------------------ 1 file changed, 169 deletions(-) delete mode 100755 landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py diff --git a/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py b/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py deleted file mode 100755 index 226af4ad6..000000000 --- a/landice/output_processing_li/ismip7_postprocessing/recalculate_missing_2d_state_vars.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python - -""" -This script copies a restart file of a MALI simulation -and re-calculates missing state variables for a -missing time level and writes them to an updated restart file. -""" - -import argparse -import os -import xarray as xr -import numpy as np - - -def main(): - parser = argparse.ArgumentParser( - description='process MALI outputs for the ISMIP7' - 'submission') - parser.add_argument("-f", "--file", dest="file_in", - required=True, - help="restart file to be read in") - parser.add_argument("-o", "--output_file", dest="file_out", - required=True, - help="output file name") - parser.add_argument("-p", "--output_file_path", - dest="output_path") - - args = parser.parse_args() - - # read in a restart file that needs to be re-written - if args.file_in is None: - print("--- restart file is not provided. Aborting... ---") - else: - print("\n--- Reading in the restart file ---") - - file_in = xr.open_dataset( - args.file_in, - decode_times=False, - decode_cf=False) - - # get needed info from restart file - cellMask = file_in['cellMask'][:, :] - thickness = file_in['thickness'][:, :] - bedTopography = file_in['bedTopography'][:, :] - sfcAirTemp = file_in['surfaceAirTemperature'][:, :] - uReconstructX = file_in['uReconstructX'][:, :, :] - uReconstructY = file_in['uReconstructY'][:, :, :] - layerThicknessFractions = file_in['layerThicknessFractions'] - nTime = file_in.dims['Time'] - nCells = file_in.dims['nCells'] - nVertLevels = file_in.dims['nVertLevels'] - - # xtime needs some massaging so xarray does not mangle it - xtime = file_in['xtime'] - xtimeStr = xtime.data.tobytes().decode() # convert to str - xtime2 = xr.DataArray( - np.array([xtimeStr], dtype=np.dtype(('S', 64))), - dims=['Time']) # convert back to char array - # followed example here: https://github.com/pydata/xarray/issues/3407 - - floating_iceMask = (cellMask[:, :] & 4) // 4 - seaLevel = 0.0 - rhoi = 910.0 - rhoo = 1028.0 - - print(f'nTime={nTime}, nCells={nCells}') - - layerInterfaceFractions = np.zeros(nVertLevels + 1, dtype=float) - lowerSfc = np.zeros([nTime, nCells], dtype=float) - upperSfc = np.zeros([nTime, nCells], dtype=float) - sfcTemp = np.zeros([nTime, nCells], dtype=float) - xvelmean = np.zeros([nTime, nCells], dtype=float) - yvelmean = np.zeros([nTime, nCells], dtype=float) - # Needed in the file so ncrcat will work, but processing will not use - # values, so can leave as zeros - surfaceSpeed = np.zeros([nTime, nCells], dtype=float) - vonMisesStress = np.zeros([nTime, nCells], dtype=float) - deltat = np.zeros([nTime,], dtype=float) - daysSinceStart = np.zeros([nTime,], dtype=float) - - print("\n--- calculating the missing state variables ---") - - # layerInterfaceFractions are the fraction associated with each - # interface - layerInterfaceFractions[0] = 0.5 * layerThicknessFractions[0] - for k in range(1, nVertLevels): - layerInterfaceFractions[k] = 0.5 * (layerThicknessFractions[k - 1] - + layerThicknessFractions[k]) - layerInterfaceFractions[nVertLevels] = 0.5 * \ - layerThicknessFractions[nVertLevels - 1] - print("layerThicknessFractions:", layerThicknessFractions[:].data) - print("layerInterfaceFractions:", layerInterfaceFractions) - - for i in range(nTime): - # calculate surface temperature (unit in Kelvin) - # 0 celsius = 273 Kelvin - sfcTemp[i, :] = np.minimum(273.15, sfcAirTemp[i, :]) - print('surfaceTemperature processed') - - lowerSfc[i, :] = np.where( - floating_iceMask, - seaLevel - thickness[i, :] * (rhoi / rhoo), - bedTopography[i, :], - ) - upperSfc[i, :] = lowerSfc[i, :] + thickness[i, :] - print('lower/upperSurface processed') - - xvelmean[i, :] = np.sum( - uReconstructX[i, :, :] * layerInterfaceFractions[:], axis=1) - yvelmean[i, :] = np.sum( - uReconstructY[i, :, :] * layerInterfaceFractions[:], axis=1) - print('x/yvelmean processed') - - # create variable dictionary of fields to include in the new file - # Note: ncrcat does not require that time-independent fields be in both - # files, so we don't need to include them in the new file. - out_data_vars = { - 'lowerSurface': (['Time', 'nCells'], lowerSfc), - 'upperSurface': (['Time', 'nCells'], upperSfc), - 'surfaceTemperature': (['Time', 'nCells'], sfcTemp), - 'xvelmean': (['Time', 'nCells'], xvelmean), - 'yvelmean': (['Time', 'nCells'], yvelmean), - 'surfaceSpeed': (['Time', 'nCells'], surfaceSpeed), - 'vonMisesStress': (['Time', 'nCells'], vonMisesStress), - 'deltat': (['Time',], deltat), - 'daysSinceStart': (['Time',], daysSinceStart), - 'xtime': xtime2 - } - # create xarray dataset object - dataOut = xr.Dataset(data_vars=out_data_vars) - # another hacky thing to make xarray handle xtime correctly - dataOut.xtime.encoding.update({"char_dim_name": "StrLen"}) - # learned this from: https://github.com/pydata/xarray/issues/2895 - - print("\n--- copying over unmodified variables from the restart " - "file ---") - for var in [ - 'thickness', - 'uReconstructX', - 'uReconstructY', - 'bedTopography', - 'basalTemperature', - 'betaSolve', - 'cellMask', - 'damage', - ]: - print(" Copying", var) - dataOut[var] = file_in[var] - - # save/write out the new file - # define the path to which the output (processed) files will be saved - if args.output_path is None: - output_path = os.getcwd() - else: - output_path = args.output_path - - if not os.path.isdir(output_path): - os.makedirs(output_path) - - print(f"file output path: {output_path}") - file_out_path = os.path.join(output_path, args.file_out) - dataOut.to_netcdf(file_out_path, mode='w', unlimited_dims=['Time']) - file_in.close() - - print("\n--- process complete! ---") - - -if __name__ == "__main__": - main() From c2dcc895f3a55bb458e804d1fd39f69691ee15fb Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 13:28:27 -0700 Subject: [PATCH 28/33] ISMIP7 grid file is required --- .../ismip7_postprocessing/post_process_mali_to_ismip7.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index e9a9e5bb8..9960c1628 100644 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -39,6 +39,7 @@ -m /pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/relaxed_10yrs_4km.nc \ -s "/pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/output/output_state_*.nc" \ -f "/pscratch/sd/t/trhille/ISMIP7/AIS_runs/no_slm_20260616/landice/ismip7_run/ismip7_ais/historical_CESM2-WACCM/output/output_state_*.nc" \ + --ismip7_grid_file /pscratch/sd/h/hoffman2/ISMIP7-postprocessing-June30-deadline/AIS/misc/af2_AIS_04000m_v1.nc \ --reuse_mapping_file mapping_mali_to_ismip7.conserve.20260626T130744.nc """ @@ -130,6 +131,7 @@ def main(): parser.add_argument( "--ismip7_grid_file", dest="ismip7_grid_file", + required=True, help="Input ismip7 mesh file.", ) parser.add_argument( @@ -203,10 +205,7 @@ def main(): args.input_state_pattern is not None or args.input_flux_pattern is not None): - # If the user has not provided a mapping file to reuse, check that the - # ismip7 grid file is provided and valid - if args.reuse_mapping_file is None: - check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) + check_ismip7_grid_file(args.ismip7_grid_file, args.res_ismip7_grid) method_remap = args.method_remap if args.reuse_mapping_file is not None: From 1923c6b1219c63ee5e527f424a85976d1e9192ff Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 13:31:20 -0700 Subject: [PATCH 29/33] Fix np.nan capitalization --- .../ismip7_postprocessing/post_process_mali_to_ismip7.py | 2 +- .../ismip7_postprocessing/process_flux_variables_ismip7.py | 6 +++--- .../ismip7_postprocessing/process_state_variables_ismip7.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py index 9960c1628..5fb0bd02e 100644 --- a/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/post_process_mali_to_ismip7.py @@ -63,7 +63,7 @@ from process_state_variables_ismip7 import process_state_pipeline DEFAULT_AUTHORS = 'Matthew Hoffman, Trevor Hillebrand, Holly Kyeore Han' -DEFAULT_GROUP = 'Los Alamos National Laboratory, Department of Energy' +DEFAULT_GROUP = 'Los Alamos National Laboratory, U.S. Department of Energy' DEFAULT_MODEL = 'MALI (MPAS-Albany Land Ice model)' DEFAULT_GROUP_NICKNAME = 'DOE' diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index 0591cb771..2bb7756c3 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -86,7 +86,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, data.close() return var_mali = data.variables[mali_var_name][:, :, :] - var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN + var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.nan timeSteps, latN, lonN = np.shape(var_mali) dataOut = Dataset( @@ -102,7 +102,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, dataOut.createDimension('x', lonN) dataOut.createDimension('y', latN) dataValues = dataOut.createVariable(ismip7_var_name, 'd', - ('time', 'y', 'x'), fill_value=np.NAN) + ('time', 'y', 'x'), fill_value=np.nan) xValues = dataOut.createVariable('x', 'd', ('x')) yValues = dataOut.createVariable('y', 'd', ('y')) timeValues = dataOut.createVariable('time', 'd', ('time')) @@ -110,7 +110,7 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, for i in range(timeSteps): mask = iceMask[i, :, :] tmp = var_mali[i, :, :] - tmp[mask == 0] = np.NAN + tmp[mask == 0] = np.nan dataValues[i, :, :] = tmp timeValues[i] = (timeBndsMin[i] + timeBndsMax[i]) / 2.0 timebndsValues[i, 0] = timeBndsMin[i] diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 0a474b2bf..d54fed9bc 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -127,7 +127,7 @@ def write_netcdf_2d_state_vars( var_sftgrf = data.variables['sftgrf'][:, :, :] var_sftflf = data.variables['sftflf'][:, :, :] var_mali = data.variables[mali_var_name][:, :, :] - var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.NAN + var_mali[np.where(abs(var_mali + 1e34) < 1e33)] = np.nan timeSteps, latN, lonN = np.shape(var_mali) dataOut = Dataset( @@ -141,7 +141,7 @@ def write_netcdf_2d_state_vars( dataOut.createDimension('x', lonN) dataOut.createDimension('y', latN) dataValues = dataOut.createVariable(ismip7_var_name, 'd', - ('time', 'y', 'x'), fill_value=np.NAN) + ('time', 'y', 'x'), fill_value=np.nan) xValues = dataOut.createVariable('x', 'd', ('x')) yValues = dataOut.createVariable('y', 'd', ('y')) timeValues = dataOut.createVariable('time', 'd', ('time')) @@ -159,7 +159,7 @@ def write_netcdf_2d_state_vars( else: mask = var_sftgif[i, :, :] tmp = var_mali[i, :, :] - tmp[mask == 0] = np.NAN + tmp[mask == 0] = np.nan dataValues[i, :, :] = tmp for i in range(latN): From cae42c6ed2f487dd709fba160b037368d0617d13 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 14:22:32 -0700 Subject: [PATCH 30/33] Clean up to flux processing Fixes a number of small problems and general cleanup --- .../process_flux_variables_ismip7.py | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index 2bb7756c3..a83b95d97 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -12,10 +12,10 @@ EXPECTED_FLUX_VARIABLES = [ - 'daysSinceStart', 'simulationStartTime', - 'timeBndsMin', 'timeBndsMax', 'iceMask', + 'daysSinceStart', 'simulationStartTime', 'cellMask', 'avgSMBFlux', 'avgFloatingBMBFlux', 'avgGroundedBMBFlux', - 'avgDhdt', 'avgCalvingFlux', 'avgGroundingLineFlux', + 'avgDhdt', 'avgCalvingFlux', 'avgFaceMeltFlux', + 'avgGroundingLineFlux', ] @@ -30,7 +30,9 @@ def check_flux_files(files): def process_flux_vars(files, tmp_file): """ - Concatenate/prepare flux files into a temporary file for remapping. + Prepare flux files into a temporary file for remapping. + This is very simple, because flux variables are already + time-averaged by MALI. Parameters ---------- @@ -50,7 +52,11 @@ def process_flux_vars(files, tmp_file): 'daysSinceStart' in ds_flux and 'units' in ds_flux['daysSinceStart'].attrs): del ds_flux['daysSinceStart'].attrs['units'] - ds_flux.to_netcdf(tmp_file) + + ds_flux_out = ds_flux[EXPECTED_FLUX_VARIABLES] + ds_flux_out.to_netcdf(tmp_file) + + ds_flux_out.close() ds_flux.close() @@ -75,12 +81,19 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, data = Dataset(remapped_mali_flux_file, 'r') data.set_auto_mask(False) - iceMask = data.variables['iceMask'][:, :, :] + iceMask = (data.variables['cellMask'][:, :, :] & 2) / 2 # grounded: dynamic ice simulationStartTime = data.variables['simulationStartTime'][:].tobytes( ).decode('utf-8').strip().strip('\x00') simulationStartDate = simulationStartTime.split("_")[0] - timeBndsMin = data.variables['timeBndsMin'][:] - timeBndsMax = data.variables['timeBndsMax'][:] + daysSinceStart = data.variables['daysSinceStart'][:] + refYear = int(simulationStartDate[0:4]) + decYears = refYear + daysSinceStart / 365.0 + years_flux = np.floor(decYears) + + # Flux outputs are annual means over each calendar year. + # Bounds are Jan 1 of that year through Jan 1 of the following year. + timeBndsMin = (years_flux - refYear) * 365.0 + timeBndsMax = (years_flux + 1.0 - refYear) * 365.0 if mali_var_name not in data.variables: print(f"WARNING: {mali_var_name} not present. Skipping.") data.close() @@ -157,7 +170,6 @@ def generate_output_2d_flux_vars(file_remapped_mali_flux, output_path: path to which the final output files are saved """ - print("Writing 2d flux variables") # ----------- acabf ------------------ write_netcdf_2d_flux_vars('avgSMBFlux', 'acabf', 'land_ice_surface_specific_mass_balance_flux', @@ -241,17 +253,17 @@ def process_flux_pipeline(flux_files, mapping_file, ismip7_grid_file, process_flux_vars(flux_files, tmp_flux_file) print("Remapping flux file.") - processed_file_flux = 'processed_flux.nc' + remapped_file_flux = 'remapped_flux.nc' check_call(["ncremap", "-i", tmp_flux_file, - "-o", processed_file_flux, + "-o", remapped_file_flux, "-m", mapping_file, "-P", "mpas"]) print("Writing processed and remapped flux fields to ISMIP7 file format.") - generate_output_2d_flux_vars(processed_file_flux, + generate_output_2d_flux_vars(remapped_file_flux, ismip7_grid_file, output_path, metadata) os.remove(tmp_flux_file) - os.remove(processed_file_flux) + os.remove(remapped_file_flux) From 6802b05c4649562acc3715698a3a0596168779ec Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 14:28:57 -0700 Subject: [PATCH 31/33] subset state var tmp file to only those needing remapping --- .../process_state_variables_ismip7.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index d54fed9bc..0ebba9e47 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -17,6 +17,18 @@ 'uReconstructX', 'uReconstructY', 'upperSurface', ] +# Keep only variables needed downstream by generate_output_2d_state_vars +# and write_netcdf_2d_state_vars. +REQUIRED_STATE_OUTPUT_VARIABLES = [ + 'daysSinceStart', 'simulationStartTime', + 'thickness', 'upperSurface', 'lowerSurface', 'bedTopography', + 'uReconstructX_sfc', 'uReconstructY_sfc', + 'uReconstructX_base', 'uReconstructY_base', + 'xvelmean', 'yvelmean', 'surfaceTemperature', + 'litempbotgr', 'litempbotfl', 'strbasemag', + 'sftgif', 'sftgrf', 'sftflf', +] + def check_state_files(files): """ @@ -87,7 +99,19 @@ def process_state_vars(files, tmp_file): dynamic_mask ) - inputfile_state_vars.to_netcdf(tmp_file) + missing = [ + var for var in REQUIRED_STATE_OUTPUT_VARIABLES + if var not in inputfile_state_vars + ] + if missing: + raise ValueError( + "Processed state dataset is missing required output variables: " + f"{missing}" + ) + + output_state_vars = inputfile_state_vars[REQUIRED_STATE_OUTPUT_VARIABLES] + output_state_vars.to_netcdf(tmp_file) + output_state_vars.close() inputfile_state_vars.close() From b9269916b376d20f586d77f2d06bdee964fab88f Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 15:41:33 -0700 Subject: [PATCH 32/33] Fix time indexing for flux processing --- .../process_flux_variables_ismip7.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index a83b95d97..39e2ef4b0 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -48,12 +48,17 @@ def process_flux_vars(files, tmp_file): data_vars='minimal', coords='minimal', compat='override') - if ( - 'daysSinceStart' in ds_flux and - 'units' in ds_flux['daysSinceStart'].attrs): + if 'units' in ds_flux['daysSinceStart'].attrs: del ds_flux['daysSinceStart'].attrs['units'] + # subset to only required variables to keep file small for remapping ds_flux_out = ds_flux[EXPECTED_FLUX_VARIABLES] + # remove the first time step (which is always 0) + time_mask = ~np.isclose(ds_flux_out['daysSinceStart'].values, 0.0) + if not np.any(time_mask): + raise ValueError("No flux time records remain after dropping daysSinceStart==0.") + ds_flux_out = ds_flux_out.isel(Time=time_mask) + ds_flux_out.to_netcdf(tmp_file) ds_flux_out.close() @@ -91,9 +96,9 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, years_flux = np.floor(decYears) # Flux outputs are annual means over each calendar year. - # Bounds are Jan 1 of that year through Jan 1 of the following year. - timeBndsMin = (years_flux - refYear) * 365.0 - timeBndsMax = (years_flux + 1.0 - refYear) * 365.0 + # Bounds are Jan 1 of the previous year through Jan 1 of the year indexed. + timeBndsMin = (years_flux - refYear - 1.0) * 365.0 + timeBndsMax = (years_flux - refYear) * 365.0 if mali_var_name not in data.variables: print(f"WARNING: {mali_var_name} not present. Skipping.") data.close() From bb7eb25486667798cb0ed5f22d49d269edab4cd2 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 26 Jun 2026 21:21:47 -0700 Subject: [PATCH 33/33] Fix masking for flux vars I set up masks that could be used for flux vars, but then I disabled using them altogether. This is because the masks at the end of the year will not necessarily be consistent with where fluxes were applied during the year. It makes more sense to me to leave fluxes unmasked and just have 0 values for the flux at places where they did not occur. --- .../process_flux_variables_ismip7.py | 41 +++++++++++++++++-- .../process_state_variables_ismip7.py | 4 +- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py index 39e2ef4b0..0b9b75c80 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_flux_variables_ismip7.py @@ -18,6 +18,16 @@ 'avgGroundingLineFlux', ] +REQUIRED_FLUX_REMAPPING_VARIABLES = [ + 'daysSinceStart', 'simulationStartTime', + 'avgSMBFlux', 'avgFloatingBMBFlux', 'avgGroundedBMBFlux', + 'avgDhdt', 'avgCalvingFlux', 'avgFaceMeltFlux', + 'avgGroundingLineFlux', + # masks not currently used - see note below in process_flux_vars + #'ice_mask', 'floating_mask', 'dynamic_mask', 'grounded_mask', + #'gl_mask' +] + def check_flux_files(files): """ @@ -51,8 +61,31 @@ def process_flux_vars(files, tmp_file): if 'units' in ds_flux['daysSinceStart'].attrs: del ds_flux['daysSinceStart'].attrs['units'] + # variables that need to be modified prior to remapping + # these masks are not currently used, but would be needed if the + # flux vars are meant to be masked. + # However there is a potential issue that fluxes could have occurred + # in locations that fall outside of the final mask, because + # the fluxes are time-averaged over a year, and the mask is only + # valid at the end of the year. + #ds_flux['ice_mask'] = (ds_flux['cellMask'][:, :] & 2) / 2 # grounded: dynamic ice + #ds_flux['floating_mask'] = (ds_flux['cellMask'][:, :] & 4) / 4 + #ds_flux['dynamic_mask'] = (ds_flux['cellMask'][:, :] & 2) / 2 + #ds_flux['grounded_mask'] = (ds_flux['cellMask'][:, :] * 0 + 1) - ds_flux['floating_mask'] + #ds_flux['gl_mask'] = (ds_flux['cellMask'][:, :] & 256) / 256 + + missing = [ + var for var in REQUIRED_FLUX_REMAPPING_VARIABLES + if var not in ds_flux + ] + if missing: + raise ValueError( + "Processed flux dataset is missing required output variables: " + f"{missing}" + ) + # subset to only required variables to keep file small for remapping - ds_flux_out = ds_flux[EXPECTED_FLUX_VARIABLES] + ds_flux_out = ds_flux[REQUIRED_FLUX_REMAPPING_VARIABLES] # remove the first time step (which is always 0) time_mask = ~np.isclose(ds_flux_out['daysSinceStart'].values, 0.0) if not np.any(time_mask): @@ -86,7 +119,6 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, data = Dataset(remapped_mali_flux_file, 'r') data.set_auto_mask(False) - iceMask = (data.variables['cellMask'][:, :, :] & 2) / 2 # grounded: dynamic ice simulationStartTime = data.variables['simulationStartTime'][:].tobytes( ).decode('utf-8').strip().strip('\x00') simulationStartDate = simulationStartTime.split("_")[0] @@ -126,9 +158,10 @@ def write_netcdf_2d_flux_vars(mali_var_name, ismip7_var_name, var_std_name, timeValues = dataOut.createVariable('time', 'd', ('time')) for i in range(timeSteps): - mask = iceMask[i, :, :] + # mask not currently used - see note above in process_flux_vars + #mask = data.variables['ice_mask'][i, :, :] tmp = var_mali[i, :, :] - tmp[mask == 0] = np.nan + #tmp[mask == 0] = np.nan dataValues[i, :, :] = tmp timeValues[i] = (timeBndsMin[i] + timeBndsMax[i]) / 2.0 timebndsValues[i, 0] = timeBndsMin[i] diff --git a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py index 0ebba9e47..ef6166e94 100644 --- a/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py +++ b/landice/output_processing_li/ismip7_postprocessing/process_state_variables_ismip7.py @@ -19,7 +19,7 @@ # Keep only variables needed downstream by generate_output_2d_state_vars # and write_netcdf_2d_state_vars. -REQUIRED_STATE_OUTPUT_VARIABLES = [ +REQUIRED_STATE_REMAPPING_VARIABLES = [ 'daysSinceStart', 'simulationStartTime', 'thickness', 'upperSurface', 'lowerSurface', 'bedTopography', 'uReconstructX_sfc', 'uReconstructY_sfc', @@ -100,7 +100,7 @@ def process_state_vars(files, tmp_file): ) missing = [ - var for var in REQUIRED_STATE_OUTPUT_VARIABLES + var for var in REQUIRED_STATE_REMAPPING_VARIABLES if var not in inputfile_state_vars ] if missing: