HPSCTerrSys · s-poll · Jul 16, 2026
diff --git a/ctl/control_tsmp2.sh b/ctl/control_tsmp2.sh
@@ -67,8 +67,11 @@ elif [ "${SYSTEMNAME}" == "jupiter" ]; then
 check_var_def npnode 288 "Taking user setting for npnode "
 check_var_def partition booster "Taking user setting and partition "
 else
-if ( [ -z $npnode] | [ -z $partition ] ); then
-echo "No npnode and/or partition for machine '$SYSTEMNAME'. Valid machine defaults for juwels/jurecadc/jusuf."
+if [ -z "$npnode" ]; then
+echo "No npnode for machine '$SYSTEMNAME'. Please set npnode manually in master.conf. Valid machine defaults for juwels/jurecadc/jusuf/jupiter."
+fi
+if [ -z "$partition" ] && [ "${scheduler:-slurm}" != "local" ]; then
+echo "No partition/queue for machine '$SYSTEMNAME'. Please set partition manually in master.conf. Valid machine defaults for juwels/jurecadc/jusuf/jupiter."
 fi
 fi
 account_def=${BUDGET_ACCOUNTS:-slts}
@@ -77,16 +80,28 @@ check_var_def tsmp2_dir $(realpath  ${ctl_dir}/../src/TSMP2) "Taking TSMP2 defau
 check_var_def tsmp2_install_dir ${tsmp2_dir}/bin/${SYSTEMNAME^^}_${MODEL_ID} \
               "Taking TSMP2 component binaries from default dir at "
 check_var_def tsmp2_env $(find ${tsmp2_install_dir}/ -type f -name "*mpi") "Using environment file "
+check_var_def scheduler slurm "Using job scheduler="
+
+# generic job-submission string (scheduler-specific: sbatch or qsub options)
+if [[ "${scheduler}" == "pbs" ]]; then
+  jobgenstring="-A ${account} \
+                -q ${partition} \
+                -m $(sched_pbs_mailcode) \
+                ${mailaddress:+-M ${mailaddress}} \
+                -V"
+elif [[ "${scheduler}" == "local" ]]; then
+  # no queue, no account/partition/mail options: sched_submit runs jobs directly
+  jobgenstring=""
+else
+  jobgenstring="--export=ALL \
+                --account=${account} \
+                --partition=${partition} \
+                --mail-type=${mailtype} \
+                --mail-user=${mailaddress} \
+                ${reservation:+--reservation=${reservation}}"
+fi
 
-# generic sbatch string
-jobgenstring="--export=ALL \
-              --account=${account} \
-              --partition=${partition} \
-              --mail-type=${mailtype} \
-              --mail-user=${mailaddress} \
-              --reservation=${reservation}"
-
-# convert arrays to string for slurm job script
+# convert arrays to string for job script
 lprestr="${lpre[@]}"
 lsimstr="${lsim[@]}"
 lposstr="${lpos[@]}"
@@ -129,18 +144,13 @@ echo "==="
 # check if any is true
 if [[ ${lpre[*]} =~ true ]]; then
 
-jobprestring="${jobgenstring} \
-              --job-name="${expid}_${caseid}pre_${dateshort}" \
-              --time=${pre_wallclock} \
-              --output="${log_dir}/%x_%j.out" \
-              --error="${log_dir}/%x_%j.err" \
-              --nodes=1 \
-              --ntasks=${npnode}"
+jobname_pre="${expid}_${caseid}pre_${dateshort}"
+jobprestring="${jobgenstring} $(sched_step_opts "${jobname_pre}" "${pre_wallclock}" 1 "${npnode}")"
 
 # Submit to pre.job
 if (! ${debugmode}) ; then
   # Submit to sim.job
-  submit_pre=$(sbatch ${jobprestring} ${ctl_dir}/pre_ctl/pre.job 2>&1)
+  submit_pre=$(sched_submit "${jobname_pre}" "${jobprestring}" "${ctl_dir}/pre_ctl/pre.job")
   echo $submit_pre" for preprocessing"
 else
   # Set lpre run & cleanup to false and source pre.job
@@ -151,7 +161,7 @@ else
 fi
 
 # get jobid
-pre_id=$(echo $submit_pre | awk 'END{print $(NF)}')
+pre_id=$(sched_parse_jobid "$submit_pre")
 
 fi # $lpre
 
@@ -180,18 +190,13 @@ else
 fi # lpre
 
 #
-jobsimstring="${jobgenstring} \
-              --job-name="${expid}_${caseid}sim_${dateshort}" \
-              --dependency=${dependencystring} \
-              --time=${sim_wallclock} \
-              --output="${log_dir}/%x_%j.out" \
-              --error="${log_dir}/%x_%j.err" \
-              --nodes=${tot_node} \
-              --ntasks=${tot_proc}"
+jobname_sim="${expid}_${caseid}sim_${dateshort}"
+jobsimstring="${jobgenstring} $(sched_step_opts "${jobname_sim}" "${sim_wallclock}" "${tot_node}" "${tot_proc}") \
+              $(sched_dependency_opt "${dependencystring}")"
 
 if (! ${debugmode}) ; then
   # Submit to sim.job
-  submit_sim=$(sbatch ${jobsimstring} ${ctl_dir}/sim_ctl/sim.job 2>&1)
+  submit_sim=$(sched_submit "${jobname_sim}" "${jobsimstring}" "${ctl_dir}/sim_ctl/sim.job")
   echo $submit_sim" for simulation"
 else
   # Set lsim run & cleanup to false and source sim.job
@@ -202,7 +207,7 @@ else
 fi
 
 # get jobid
-sim_id=$(echo $submit_sim | awk 'END{print $(NF)}')
+sim_id=$(sched_parse_jobid "$submit_sim")
 
 fi # $lsim
 
@@ -221,20 +226,16 @@ else
 fi
 
 # Configure TSMP2 Postprocessing
-jobposstring="${jobgenstring} \
-              --job-name="${expid}_${caseid}pos_${dateshort}" \
-              --time=${pos_wallclock} \
-              --output="${log_dir}/%x_%j.out" \
-              --error="${log_dir}/%x_%j.err" \
-              --nodes=1 \
-              --ntasks=${npnode}"
+jobname_pos="${expid}_${caseid}pos_${dateshort}"
+jobposstring="${jobgenstring} $(sched_step_opts "${jobname_pos}" "${pos_wallclock}" 1 "${npnode}") \
+              $(sched_dependency_opt "${dependencystring}")"
 
 # Submit to pos.job
-submit_pos=$(sbatch ${jobposstring} ${ctl_dir}/pos_ctl/pos.job 2>&1)
+submit_pos=$(sched_submit "${jobname_pos}" "${jobposstring}" "${ctl_dir}/pos_ctl/pos.job")
 echo $submit_pos" for postprocessing"
 
 # get jobid
-pos_id=$(echo $submit_pos | awk 'END{print $(NF)}')
+pos_id=$(sched_parse_jobid "$submit_pos")
 
 fi # $lpos
 
@@ -253,20 +254,16 @@ else
 fi
 
 # Configure TSMP2 Postprocessing
-jobvisstring="${jobgenstring} \
-              --job-name="${expid}_${caseid}vis_${dateshort}" \
-              --time=${vis_wallclock} \
-              --output="${log_dir}/%x_%j.out" \
-              --error="${log_dir}/%x_%j.err" \
-              --nodes=1 \
-              --ntasks=${npnode}"
+jobname_vis="${expid}_${caseid}vis_${dateshort}"
+jobvisstring="${jobgenstring} $(sched_step_opts "${jobname_vis}" "${vis_wallclock}" 1 "${npnode}") \
+              $(sched_dependency_opt "${dependencystring}")"
 
 # Submit to vis.job
-submit_vis=$(sbatch ${jobvisstring} ${ctl_dir}/vis_ctl/vis.job 2>&1)
+submit_vis=$(sched_submit "${jobname_vis}" "${jobvisstring}" "${ctl_dir}/vis_ctl/vis.job")
 echo $submit_vis" for visualization"
 
 # get jobid
-vis_id=$(echo $submit_vis | awk 'END{print $(NF)}')
+vis_id=$(sched_parse_jobid "$submit_vis")
 
 fi # $lvis
 

diff --git a/ctl/master.conf b/ctl/master.conf
@@ -8,6 +8,9 @@ EXP_ID="eur-12-iic" #"eur-11u"
 CASE_ID="" # identifier for cases
 conf_file="" # configure file for cases, also possible ${EXP_ID}.conf
 
+# job scheduler: slurm (default), pbs, or local
+scheduler=slurm # 'local' runs jobs synchronously in the foreground with no queue
+
 # main switches (PREprocessing, SIMulations, POSt-processing, VISualisation)
 lpre=( false false false ) # config, run, cleanup
 lsim=( true true true ) # config, run, cleanup
@@ -22,14 +25,14 @@ startdate="2017-07-01T00:00Z" # ISO norm 8601
 inidate=${startdate} # ISO norm 8601, fix for entire simulation
 numsimstep=1 # number of simulation steps, simulation period = numsimstep * simlength
 
-# mail notification for slurm jobs
-mailtype=NONE # NONE, BEGIN, END, FAIL, REQUEUE, ALL
+# mail notification for scheduler jobs
+mailtype=NONE # NONE, BEGIN, END, FAIL, REQUEUE, ALL (mapped to PBS -m code if scheduler=pbs)
 mailaddress=""
 
 # user setting, leave empty for jsc machine defaults
 prevjobid="" # previous job-id, default leave empty
 npnode="" # number of cores per node
-partition="" # compute partition
+partition="" # compute partition (slurm) / queue (pbs)
 account="" # SET compute account. $BUDGET_ACCOUNTS / slts is used, if not set.
 
 # wallclock

diff --git a/ctl/pos_ctl/pos.job b/ctl/pos_ctl/pos.job
@@ -3,6 +3,7 @@
 ###
 # Import functions
 ###
+source ${ctl_dir}/utils_tsmp2.sh
 source ${ctl_dir}/pos_ctl/pos_config.sh
 source ${ctl_dir}/pos_ctl/pos_run.sh
 source ${ctl_dir}/pos_ctl/pos_cleanup.sh

diff --git a/ctl/pre_ctl/pre.job b/ctl/pre_ctl/pre.job
@@ -3,6 +3,7 @@
 ###
 # Import functions
 ###
+source ${ctl_dir}/utils_tsmp2.sh
 source ${ctl_dir}/pre_ctl/pre_config.sh
 source ${ctl_dir}/pre_ctl/pre_run.sh
 source ${ctl_dir}/pre_ctl/pre_cleanup.sh

diff --git a/ctl/pre_ctl/pre_cleanup.sh b/ctl/pre_ctl/pre_cleanup.sh
@@ -6,6 +6,9 @@ pre_cleanup(){
 
 echo "Start Pre-processing cleanup"
 
+job_id=$(sched_job_id)
+job_name=$(sched_job_name)
+
 ####################
 # CLM
 ####################
@@ -23,7 +26,11 @@ fi
 
 # move logs
 mkdir -p ${eclmfrc_dir}/log/
-mv ${log_dir}/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.{err,out} ${eclmfrc_dir}/log/.
+if [[ "${scheduler}" == "pbs" ]]; then
+  mv ${log_dir}/${job_name}.{e,o}${job_id} ${eclmfrc_dir}/log/${job_name}_${job_id}.out
+else
+  mv ${log_dir}/${job_name}_${job_id}.{err,out} ${eclmfrc_dir}/log/.
+fi
 
 # remove working directory
 rm -rf ${pre_dir:?}

diff --git a/ctl/pre_ctl/pre_config.sh b/ctl/pre_ctl/pre_config.sh
@@ -61,7 +61,7 @@ if ${debugmode}; then
 
 # create job submission script (pre.job)
 echo "#!/usr/bin/env bash" > pre.job
-echo "#SBATCH ${jobprestring//[$'\t\r\n']}" >> pre.job
+echo "$(sched_directive_prefix) ${jobprestring//[$'\t\r\n']}" >> pre.job
 
 # add modelid, which is needed
 echo "" >> pre.job

diff --git a/ctl/pre_ctl/pre_run.sh b/ctl/pre_ctl/pre_run.sh
@@ -24,7 +24,7 @@ for yyyymm in "${listfrcfile[@]}"; do
    mkdir -p $yyyymm # temporary directory for scripts
 
    # extract data from ERA5 meteocloud
-   srun --exclusive -n 1 ${lsmforcgensrc_dir}/extract_ERA5_meteocloud.sh iyear=$year imonth=$month \
+   run_serial_step ${lsmforcgensrc_dir}/extract_ERA5_meteocloud.sh iyear=$year imonth=$month \
         outdir=${pre_dir}/${yyyymm} #quiet=y
 
 #  # needs to be done in advance
@@ -35,7 +35,7 @@ for yyyymm in "${listfrcfile[@]}"; do
    ln -s ${cdsapi_dtadir}/download_era5_${year}_$month.zip ${yyyymm}/
 
    # preparation script
-   srun --exclusive -n 1 ${lsmforcgensrc_dir}/prepare_ERA5_input.sh pathdata=${pre_dir}/${yyyymm} \
+   run_serial_step ${lsmforcgensrc_dir}/prepare_ERA5_input.sh pathdata=${pre_dir}/${yyyymm} \
 	                  iyear=$year imonth=$month wrkdir=${yyyymm}
 
 done

diff --git a/ctl/sim_ctl/sim.job b/ctl/sim_ctl/sim.job
@@ -3,6 +3,7 @@
 ###
 # Import functions
 ###
+source ${ctl_dir}/utils_tsmp2.sh
 source ${ctl_dir}/sim_ctl/sim_config.sh
 source ${ctl_dir}/sim_ctl/sim_run.sh
 source ${ctl_dir}/sim_ctl/sim_cleanup.sh

diff --git a/ctl/sim_ctl/sim_cleanup.sh b/ctl/sim_ctl/sim_cleanup.sh
@@ -6,6 +6,9 @@ echo "###"
 echo "# Cleanup Simulation"
 echo "###"
 
+job_id=$(sched_job_id)
+job_name=$(sched_job_name)
+
 parse_config_file ${conf_file} "sim_clean_general"
 
 file_op_mode=${file_op_mode:-"copy"}
@@ -70,7 +73,7 @@ if [[ "${modelid}" == *clm* ]]; then
   file_op ${sim_dir}/eCLM_*.clm2.h* ${simout_dir}/out/eclm
 
   # Model log
-  file_op ${sim_dir}/logs/${SLURM_JOB_ID}.comp_*.log ${simout_dir}/log/
+  file_op ${sim_dir}/logs/${job_id}.comp_*.log ${simout_dir}/log/
   file_op ${sim_dir}/timing/model_timing_stats ${simout_dir}/log/
 
   # Restart
@@ -111,13 +114,22 @@ if [[ "${modelid}" == *parflow* ]]; then
 
 fi # parflow
 
-file_op ${sim_dir}/slm_multiprog_mapping.conf ${simout_dir}/log/
+file_op ${sim_dir}/${mpmd_mapping_file} ${simout_dir}/log/
 
 # sim logs
-mv ${log_dir}/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.{err,out} ${simout_dir}/log/.
+if [[ "${scheduler}" == "pbs" ]]; then
+  mv ${log_dir}/${job_name}.{e,o}${job_id} ${simout_dir}/log/${job_name}_${job_id}.out
+else
+  mv ${log_dir}/${job_name}_${job_id}.{err,out} ${simout_dir}/log/.
+fi
 
 # job info
-echo $(scontrol show job ${SLURM_JOB_ID}) > ${simout_dir}/log/job_info.log
+case "${scheduler}" in
+  slurm) echo $(scontrol show job ${job_id}) > ${simout_dir}/log/job_info.log ;;
+  pbs)   qstat -f ${job_id} > ${simout_dir}/log/job_info.log 2>&1 ;;
+  local) echo "local job ${job_name} (id ${job_id}) ran on $(hostname) at $(date)" > ${simout_dir}/log/job_info.log ;;
+  *)     echo $(scontrol show job ${job_id}) > ${simout_dir}/log/job_info.log ;;
+esac
 
 # remove run directory
 rm -rf ${sim_dir:?}

diff --git a/ctl/sim_ctl/sim_config.sh b/ctl/sim_ctl/sim_config.sh
@@ -23,21 +23,37 @@ if [ -e "${sim_dir}" ]; then
 fi
 mkdir -p $sim_dir
 
-# slm_multiprog
-if [[ "${modelid}" == *icon* ]]; then
-   echo "0-__icon_pe__   ./icon" >> ${sim_dir}/slm_multiprog_mapping.conf
-fi
-if [[ "${modelid}" == *eclm* ]]; then
-   echo "__clm_ps__-__clm_pe__ ./eclm" >> ${sim_dir}/slm_multiprog_mapping.conf
-fi
-if [[ "${modelid}" == *parflow* ]]; then
-   echo "__pfl_ps__-__pfl_pe__ ./parflow __pfl_expid__" >>  ${sim_dir}/slm_multiprog_mapping.conf
+# MPMD program-rank mapping file for the coupled run (format depends on scheduler)
+if [[ "${scheduler}" == "pbs" || "${scheduler}" == "local" ]]; then
+   # Open MPI app-context file, ranks assigned per-line count (used with 'mpiexec --app')
+   mpmd_mapping_file=mpi_multiprog_mapping.conf
+   if [[ "${modelid}" == *icon* ]]; then
+      echo "-np ${ico_proc} ./icon" >> ${sim_dir}/${mpmd_mapping_file}
+   fi
+   if [[ "${modelid}" == *eclm* ]]; then
+      echo "-np ${clm_proc} ./eclm" >> ${sim_dir}/${mpmd_mapping_file}
+   fi
+   if [[ "${modelid}" == *parflow* ]]; then
+      echo "-np ${pfl_proc} ./parflow __pfl_expid__" >> ${sim_dir}/${mpmd_mapping_file}
+   fi
+else
+   # Slurm multi-prog rank-range mapping file (used with 'srun --multi-prog')
+   mpmd_mapping_file=slm_multiprog_mapping.conf
+   if [[ "${modelid}" == *icon* ]]; then
+      echo "0-__icon_pe__   ./icon" >> ${sim_dir}/${mpmd_mapping_file}
+   fi
+   if [[ "${modelid}" == *eclm* ]]; then
+      echo "__clm_ps__-__clm_pe__ ./eclm" >> ${sim_dir}/${mpmd_mapping_file}
+   fi
+   if [[ "${modelid}" == *parflow* ]]; then
+      echo "__pfl_ps__-__pfl_pe__ ./parflow __pfl_expid__" >>  ${sim_dir}/${mpmd_mapping_file}
+   fi
+   sed -i "s/__icon_pe__/$(($ico_proc-1))/" ${sim_dir}/${mpmd_mapping_file}
+   sed -i "s/__clm_ps__/$(($ico_proc))/" ${sim_dir}/${mpmd_mapping_file}
+   sed -i "s/__clm_pe__/$(($ico_proc+$clm_proc-1))/" ${sim_dir}/${mpmd_mapping_file}
+   sed -i "s/__pfl_ps__/$(($ico_proc+$clm_proc))/" ${sim_dir}/${mpmd_mapping_file}
+   sed -i "s/__pfl_pe__/$(($ico_proc+$clm_proc+$pfl_proc-1))/" ${sim_dir}/${mpmd_mapping_file}
 fi
-sed -i "s/__icon_pe__/$(($ico_proc-1))/" ${sim_dir}/slm_multiprog_mapping.conf
-sed -i "s/__clm_ps__/$(($ico_proc))/" ${sim_dir}/slm_multiprog_mapping.conf
-sed -i "s/__clm_pe__/$(($ico_proc+$clm_proc-1))/" ${sim_dir}/slm_multiprog_mapping.conf
-sed -i "s/__pfl_ps__/$(($ico_proc+$clm_proc))/" ${sim_dir}/slm_multiprog_mapping.conf
-sed -i "s/__pfl_pe__/$(($ico_proc+$clm_proc+$pfl_proc-1))/" ${sim_dir}/slm_multiprog_mapping.conf
 
 # change to run directory
 cd ${sim_dir}
@@ -259,7 +275,7 @@ if [[ "${modelid}" == *parflow* ]]; then
   sed -i "s#__inifile__#$(basename "$fini_pfl")#" coup_oas.tcl
   sed -i "s/__pfltsfilerst__/${pfltsfilerst}/" coup_oas.tcl
   sed -i "s/__pfloutmfilt__/${pfloutmfilt}/" coup_oas.tcl
-  sed -i "s/__pfl_expid__/$EXP_ID/" slm_multiprog_mapping.conf
+  sed -i "s/__pfl_expid__/$EXP_ID/" ${mpmd_mapping_file}
 
   # --- execute ParFlow distributeing tcl-scripts
   export PARFLOW_DIR=${tsmp2_install_dir}
@@ -305,7 +321,7 @@ if ${debugmode}; then
 
 # create job submission script (tsmp2.job)
 echo "#!/usr/bin/env bash" > tsmp2.job
-echo "#SBATCH ${jobsimstring//[$'\t\r\n']}" >> tsmp2.job
+echo "$(sched_directive_prefix) ${jobsimstring//[$'\t\r\n']}" >> tsmp2.job
 
 # add modelid, which is needed
 echo "" >> tsmp2.job