diff --git a/ctl/control_tsmp2.sh b/ctl/control_tsmp2.sh index 31b3cb8..138a4a7 100755 --- a/ctl/control_tsmp2.sh +++ b/ctl/control_tsmp2.sh @@ -67,8 +67,11 @@ elif [ "${SYSTEMNAME}" == "jupiter" ]; then check_var_def npnode 288 "Taking user setting for npnode " check_var_def partition booster "Taking user setting and partition " else -if ( [ -z $npnode] | [ -z $partition ] ); then -echo "No npnode and/or partition for machine '$SYSTEMNAME'. Valid machine defaults for juwels/jurecadc/jusuf." +if [ -z "$npnode" ]; then +echo "No npnode for machine '$SYSTEMNAME'. Please set npnode manually in master.conf. Valid machine defaults for juwels/jurecadc/jusuf/jupiter." +fi +if [ -z "$partition" ] && [ "${scheduler:-slurm}" != "local" ]; then +echo "No partition/queue for machine '$SYSTEMNAME'. Please set partition manually in master.conf. Valid machine defaults for juwels/jurecadc/jusuf/jupiter." fi fi account_def=${BUDGET_ACCOUNTS:-slts} @@ -77,16 +80,28 @@ check_var_def tsmp2_dir $(realpath ${ctl_dir}/../src/TSMP2) "Taking TSMP2 defau check_var_def tsmp2_install_dir ${tsmp2_dir}/bin/${SYSTEMNAME^^}_${MODEL_ID} \ "Taking TSMP2 component binaries from default dir at " check_var_def tsmp2_env $(find ${tsmp2_install_dir}/ -type f -name "*mpi") "Using environment file " +check_var_def scheduler slurm "Using job scheduler=" + +# generic job-submission string (scheduler-specific: sbatch or qsub options) +if [[ "${scheduler}" == "pbs" ]]; then + jobgenstring="-A ${account} \ + -q ${partition} \ + -m $(sched_pbs_mailcode) \ + ${mailaddress:+-M ${mailaddress}} \ + -V" +elif [[ "${scheduler}" == "local" ]]; then + # no queue, no account/partition/mail options: sched_submit runs jobs directly + jobgenstring="" +else + jobgenstring="--export=ALL \ + --account=${account} \ + --partition=${partition} \ + --mail-type=${mailtype} \ + --mail-user=${mailaddress} \ + ${reservation:+--reservation=${reservation}}" +fi -# generic sbatch string -jobgenstring="--export=ALL \ - --account=${account} \ - --partition=${partition} \ - --mail-type=${mailtype} \ - --mail-user=${mailaddress} \ - --reservation=${reservation}" - -# convert arrays to string for slurm job script +# convert arrays to string for job script lprestr="${lpre[@]}" lsimstr="${lsim[@]}" lposstr="${lpos[@]}" @@ -129,18 +144,13 @@ echo "===" # check if any is true if [[ ${lpre[*]} =~ true ]]; then -jobprestring="${jobgenstring} \ - --job-name="${expid}_${caseid}pre_${dateshort}" \ - --time=${pre_wallclock} \ - --output="${log_dir}/%x_%j.out" \ - --error="${log_dir}/%x_%j.err" \ - --nodes=1 \ - --ntasks=${npnode}" +jobname_pre="${expid}_${caseid}pre_${dateshort}" +jobprestring="${jobgenstring} $(sched_step_opts "${jobname_pre}" "${pre_wallclock}" 1 "${npnode}")" # Submit to pre.job if (! ${debugmode}) ; then # Submit to sim.job - submit_pre=$(sbatch ${jobprestring} ${ctl_dir}/pre_ctl/pre.job 2>&1) + submit_pre=$(sched_submit "${jobname_pre}" "${jobprestring}" "${ctl_dir}/pre_ctl/pre.job") echo $submit_pre" for preprocessing" else # Set lpre run & cleanup to false and source pre.job @@ -151,7 +161,7 @@ else fi # get jobid -pre_id=$(echo $submit_pre | awk 'END{print $(NF)}') +pre_id=$(sched_parse_jobid "$submit_pre") fi # $lpre @@ -180,18 +190,13 @@ else fi # lpre # -jobsimstring="${jobgenstring} \ - --job-name="${expid}_${caseid}sim_${dateshort}" \ - --dependency=${dependencystring} \ - --time=${sim_wallclock} \ - --output="${log_dir}/%x_%j.out" \ - --error="${log_dir}/%x_%j.err" \ - --nodes=${tot_node} \ - --ntasks=${tot_proc}" +jobname_sim="${expid}_${caseid}sim_${dateshort}" +jobsimstring="${jobgenstring} $(sched_step_opts "${jobname_sim}" "${sim_wallclock}" "${tot_node}" "${tot_proc}") \ + $(sched_dependency_opt "${dependencystring}")" if (! ${debugmode}) ; then # Submit to sim.job - submit_sim=$(sbatch ${jobsimstring} ${ctl_dir}/sim_ctl/sim.job 2>&1) + submit_sim=$(sched_submit "${jobname_sim}" "${jobsimstring}" "${ctl_dir}/sim_ctl/sim.job") echo $submit_sim" for simulation" else # Set lsim run & cleanup to false and source sim.job @@ -202,7 +207,7 @@ else fi # get jobid -sim_id=$(echo $submit_sim | awk 'END{print $(NF)}') +sim_id=$(sched_parse_jobid "$submit_sim") fi # $lsim @@ -221,20 +226,16 @@ else fi # Configure TSMP2 Postprocessing -jobposstring="${jobgenstring} \ - --job-name="${expid}_${caseid}pos_${dateshort}" \ - --time=${pos_wallclock} \ - --output="${log_dir}/%x_%j.out" \ - --error="${log_dir}/%x_%j.err" \ - --nodes=1 \ - --ntasks=${npnode}" +jobname_pos="${expid}_${caseid}pos_${dateshort}" +jobposstring="${jobgenstring} $(sched_step_opts "${jobname_pos}" "${pos_wallclock}" 1 "${npnode}") \ + $(sched_dependency_opt "${dependencystring}")" # Submit to pos.job -submit_pos=$(sbatch ${jobposstring} ${ctl_dir}/pos_ctl/pos.job 2>&1) +submit_pos=$(sched_submit "${jobname_pos}" "${jobposstring}" "${ctl_dir}/pos_ctl/pos.job") echo $submit_pos" for postprocessing" # get jobid -pos_id=$(echo $submit_pos | awk 'END{print $(NF)}') +pos_id=$(sched_parse_jobid "$submit_pos") fi # $lpos @@ -253,20 +254,16 @@ else fi # Configure TSMP2 Postprocessing -jobvisstring="${jobgenstring} \ - --job-name="${expid}_${caseid}vis_${dateshort}" \ - --time=${vis_wallclock} \ - --output="${log_dir}/%x_%j.out" \ - --error="${log_dir}/%x_%j.err" \ - --nodes=1 \ - --ntasks=${npnode}" +jobname_vis="${expid}_${caseid}vis_${dateshort}" +jobvisstring="${jobgenstring} $(sched_step_opts "${jobname_vis}" "${vis_wallclock}" 1 "${npnode}") \ + $(sched_dependency_opt "${dependencystring}")" # Submit to vis.job -submit_vis=$(sbatch ${jobvisstring} ${ctl_dir}/vis_ctl/vis.job 2>&1) +submit_vis=$(sched_submit "${jobname_vis}" "${jobvisstring}" "${ctl_dir}/vis_ctl/vis.job") echo $submit_vis" for visualization" # get jobid -vis_id=$(echo $submit_vis | awk 'END{print $(NF)}') +vis_id=$(sched_parse_jobid "$submit_vis") fi # $lvis diff --git a/ctl/master.conf b/ctl/master.conf index b545b93..6c675b1 100644 --- a/ctl/master.conf +++ b/ctl/master.conf @@ -8,6 +8,9 @@ EXP_ID="eur-12-iic" #"eur-11u" CASE_ID="" # identifier for cases conf_file="" # configure file for cases, also possible ${EXP_ID}.conf +# job scheduler: slurm (default), pbs, or local +scheduler=slurm # 'local' runs jobs synchronously in the foreground with no queue + # main switches (PREprocessing, SIMulations, POSt-processing, VISualisation) lpre=( false false false ) # config, run, cleanup lsim=( true true true ) # config, run, cleanup @@ -22,14 +25,14 @@ startdate="2017-07-01T00:00Z" # ISO norm 8601 inidate=${startdate} # ISO norm 8601, fix for entire simulation numsimstep=1 # number of simulation steps, simulation period = numsimstep * simlength -# mail notification for slurm jobs -mailtype=NONE # NONE, BEGIN, END, FAIL, REQUEUE, ALL +# mail notification for scheduler jobs +mailtype=NONE # NONE, BEGIN, END, FAIL, REQUEUE, ALL (mapped to PBS -m code if scheduler=pbs) mailaddress="" # user setting, leave empty for jsc machine defaults prevjobid="" # previous job-id, default leave empty npnode="" # number of cores per node -partition="" # compute partition +partition="" # compute partition (slurm) / queue (pbs) account="" # SET compute account. $BUDGET_ACCOUNTS / slts is used, if not set. # wallclock diff --git a/ctl/pos_ctl/pos.job b/ctl/pos_ctl/pos.job index be4de6f..3ca14ce 100755 --- a/ctl/pos_ctl/pos.job +++ b/ctl/pos_ctl/pos.job @@ -3,6 +3,7 @@ ### # Import functions ### +source ${ctl_dir}/utils_tsmp2.sh source ${ctl_dir}/pos_ctl/pos_config.sh source ${ctl_dir}/pos_ctl/pos_run.sh source ${ctl_dir}/pos_ctl/pos_cleanup.sh diff --git a/ctl/pre_ctl/pre.job b/ctl/pre_ctl/pre.job index 8f2cb88..61277ab 100755 --- a/ctl/pre_ctl/pre.job +++ b/ctl/pre_ctl/pre.job @@ -3,6 +3,7 @@ ### # Import functions ### +source ${ctl_dir}/utils_tsmp2.sh source ${ctl_dir}/pre_ctl/pre_config.sh source ${ctl_dir}/pre_ctl/pre_run.sh source ${ctl_dir}/pre_ctl/pre_cleanup.sh diff --git a/ctl/pre_ctl/pre_cleanup.sh b/ctl/pre_ctl/pre_cleanup.sh index fc4ecdc..053deeb 100755 --- a/ctl/pre_ctl/pre_cleanup.sh +++ b/ctl/pre_ctl/pre_cleanup.sh @@ -6,6 +6,9 @@ pre_cleanup(){ echo "Start Pre-processing cleanup" +job_id=$(sched_job_id) +job_name=$(sched_job_name) + #################### # CLM #################### @@ -23,7 +26,11 @@ fi # move logs mkdir -p ${eclmfrc_dir}/log/ -mv ${log_dir}/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.{err,out} ${eclmfrc_dir}/log/. +if [[ "${scheduler}" == "pbs" ]]; then + mv ${log_dir}/${job_name}.{e,o}${job_id} ${eclmfrc_dir}/log/${job_name}_${job_id}.out +else + mv ${log_dir}/${job_name}_${job_id}.{err,out} ${eclmfrc_dir}/log/. +fi # remove working directory rm -rf ${pre_dir:?} diff --git a/ctl/pre_ctl/pre_config.sh b/ctl/pre_ctl/pre_config.sh index 47ca63c..7cb7a97 100755 --- a/ctl/pre_ctl/pre_config.sh +++ b/ctl/pre_ctl/pre_config.sh @@ -61,7 +61,7 @@ if ${debugmode}; then # create job submission script (pre.job) echo "#!/usr/bin/env bash" > pre.job -echo "#SBATCH ${jobprestring//[$'\t\r\n']}" >> pre.job +echo "$(sched_directive_prefix) ${jobprestring//[$'\t\r\n']}" >> pre.job # add modelid, which is needed echo "" >> pre.job diff --git a/ctl/pre_ctl/pre_run.sh b/ctl/pre_ctl/pre_run.sh index 4351fa1..8a6e907 100755 --- a/ctl/pre_ctl/pre_run.sh +++ b/ctl/pre_ctl/pre_run.sh @@ -24,7 +24,7 @@ for yyyymm in "${listfrcfile[@]}"; do mkdir -p $yyyymm # temporary directory for scripts # extract data from ERA5 meteocloud - srun --exclusive -n 1 ${lsmforcgensrc_dir}/extract_ERA5_meteocloud.sh iyear=$year imonth=$month \ + run_serial_step ${lsmforcgensrc_dir}/extract_ERA5_meteocloud.sh iyear=$year imonth=$month \ outdir=${pre_dir}/${yyyymm} #quiet=y # # needs to be done in advance @@ -35,7 +35,7 @@ for yyyymm in "${listfrcfile[@]}"; do ln -s ${cdsapi_dtadir}/download_era5_${year}_$month.zip ${yyyymm}/ # preparation script - srun --exclusive -n 1 ${lsmforcgensrc_dir}/prepare_ERA5_input.sh pathdata=${pre_dir}/${yyyymm} \ + run_serial_step ${lsmforcgensrc_dir}/prepare_ERA5_input.sh pathdata=${pre_dir}/${yyyymm} \ iyear=$year imonth=$month wrkdir=${yyyymm} done diff --git a/ctl/sim_ctl/sim.job b/ctl/sim_ctl/sim.job index 87f7a05..2964dcd 100755 --- a/ctl/sim_ctl/sim.job +++ b/ctl/sim_ctl/sim.job @@ -3,6 +3,7 @@ ### # Import functions ### +source ${ctl_dir}/utils_tsmp2.sh source ${ctl_dir}/sim_ctl/sim_config.sh source ${ctl_dir}/sim_ctl/sim_run.sh source ${ctl_dir}/sim_ctl/sim_cleanup.sh diff --git a/ctl/sim_ctl/sim_cleanup.sh b/ctl/sim_ctl/sim_cleanup.sh index 98c69fe..acbc947 100755 --- a/ctl/sim_ctl/sim_cleanup.sh +++ b/ctl/sim_ctl/sim_cleanup.sh @@ -6,6 +6,9 @@ echo "###" echo "# Cleanup Simulation" echo "###" +job_id=$(sched_job_id) +job_name=$(sched_job_name) + parse_config_file ${conf_file} "sim_clean_general" file_op_mode=${file_op_mode:-"copy"} @@ -70,7 +73,7 @@ if [[ "${modelid}" == *clm* ]]; then file_op ${sim_dir}/eCLM_*.clm2.h* ${simout_dir}/out/eclm # Model log - file_op ${sim_dir}/logs/${SLURM_JOB_ID}.comp_*.log ${simout_dir}/log/ + file_op ${sim_dir}/logs/${job_id}.comp_*.log ${simout_dir}/log/ file_op ${sim_dir}/timing/model_timing_stats ${simout_dir}/log/ # Restart @@ -111,13 +114,22 @@ if [[ "${modelid}" == *parflow* ]]; then fi # parflow -file_op ${sim_dir}/slm_multiprog_mapping.conf ${simout_dir}/log/ +file_op ${sim_dir}/${mpmd_mapping_file} ${simout_dir}/log/ # sim logs -mv ${log_dir}/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.{err,out} ${simout_dir}/log/. +if [[ "${scheduler}" == "pbs" ]]; then + mv ${log_dir}/${job_name}.{e,o}${job_id} ${simout_dir}/log/${job_name}_${job_id}.out +else + mv ${log_dir}/${job_name}_${job_id}.{err,out} ${simout_dir}/log/. +fi # job info -echo $(scontrol show job ${SLURM_JOB_ID}) > ${simout_dir}/log/job_info.log +case "${scheduler}" in + slurm) echo $(scontrol show job ${job_id}) > ${simout_dir}/log/job_info.log ;; + pbs) qstat -f ${job_id} > ${simout_dir}/log/job_info.log 2>&1 ;; + local) echo "local job ${job_name} (id ${job_id}) ran on $(hostname) at $(date)" > ${simout_dir}/log/job_info.log ;; + *) echo $(scontrol show job ${job_id}) > ${simout_dir}/log/job_info.log ;; +esac # remove run directory rm -rf ${sim_dir:?} diff --git a/ctl/sim_ctl/sim_config.sh b/ctl/sim_ctl/sim_config.sh index b87cc84..1eaca32 100755 --- a/ctl/sim_ctl/sim_config.sh +++ b/ctl/sim_ctl/sim_config.sh @@ -23,21 +23,37 @@ if [ -e "${sim_dir}" ]; then fi mkdir -p $sim_dir -# slm_multiprog -if [[ "${modelid}" == *icon* ]]; then - echo "0-__icon_pe__ ./icon" >> ${sim_dir}/slm_multiprog_mapping.conf -fi -if [[ "${modelid}" == *eclm* ]]; then - echo "__clm_ps__-__clm_pe__ ./eclm" >> ${sim_dir}/slm_multiprog_mapping.conf -fi -if [[ "${modelid}" == *parflow* ]]; then - echo "__pfl_ps__-__pfl_pe__ ./parflow __pfl_expid__" >> ${sim_dir}/slm_multiprog_mapping.conf +# MPMD program-rank mapping file for the coupled run (format depends on scheduler) +if [[ "${scheduler}" == "pbs" || "${scheduler}" == "local" ]]; then + # Open MPI app-context file, ranks assigned per-line count (used with 'mpiexec --app') + mpmd_mapping_file=mpi_multiprog_mapping.conf + if [[ "${modelid}" == *icon* ]]; then + echo "-np ${ico_proc} ./icon" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *eclm* ]]; then + echo "-np ${clm_proc} ./eclm" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *parflow* ]]; then + echo "-np ${pfl_proc} ./parflow __pfl_expid__" >> ${sim_dir}/${mpmd_mapping_file} + fi +else + # Slurm multi-prog rank-range mapping file (used with 'srun --multi-prog') + mpmd_mapping_file=slm_multiprog_mapping.conf + if [[ "${modelid}" == *icon* ]]; then + echo "0-__icon_pe__ ./icon" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *eclm* ]]; then + echo "__clm_ps__-__clm_pe__ ./eclm" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *parflow* ]]; then + echo "__pfl_ps__-__pfl_pe__ ./parflow __pfl_expid__" >> ${sim_dir}/${mpmd_mapping_file} + fi + sed -i "s/__icon_pe__/$(($ico_proc-1))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__clm_ps__/$(($ico_proc))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__clm_pe__/$(($ico_proc+$clm_proc-1))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__pfl_ps__/$(($ico_proc+$clm_proc))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__pfl_pe__/$(($ico_proc+$clm_proc+$pfl_proc-1))/" ${sim_dir}/${mpmd_mapping_file} fi -sed -i "s/__icon_pe__/$(($ico_proc-1))/" ${sim_dir}/slm_multiprog_mapping.conf -sed -i "s/__clm_ps__/$(($ico_proc))/" ${sim_dir}/slm_multiprog_mapping.conf -sed -i "s/__clm_pe__/$(($ico_proc+$clm_proc-1))/" ${sim_dir}/slm_multiprog_mapping.conf -sed -i "s/__pfl_ps__/$(($ico_proc+$clm_proc))/" ${sim_dir}/slm_multiprog_mapping.conf -sed -i "s/__pfl_pe__/$(($ico_proc+$clm_proc+$pfl_proc-1))/" ${sim_dir}/slm_multiprog_mapping.conf # change to run directory cd ${sim_dir} @@ -259,7 +275,7 @@ if [[ "${modelid}" == *parflow* ]]; then sed -i "s#__inifile__#$(basename "$fini_pfl")#" coup_oas.tcl sed -i "s/__pfltsfilerst__/${pfltsfilerst}/" coup_oas.tcl sed -i "s/__pfloutmfilt__/${pfloutmfilt}/" coup_oas.tcl - sed -i "s/__pfl_expid__/$EXP_ID/" slm_multiprog_mapping.conf + sed -i "s/__pfl_expid__/$EXP_ID/" ${mpmd_mapping_file} # --- execute ParFlow distributeing tcl-scripts export PARFLOW_DIR=${tsmp2_install_dir} @@ -305,7 +321,7 @@ if ${debugmode}; then # create job submission script (tsmp2.job) echo "#!/usr/bin/env bash" > tsmp2.job -echo "#SBATCH ${jobsimstring//[$'\t\r\n']}" >> tsmp2.job +echo "$(sched_directive_prefix) ${jobsimstring//[$'\t\r\n']}" >> tsmp2.job # add modelid, which is needed echo "" >> tsmp2.job diff --git a/ctl/sim_ctl/sim_run.sh b/ctl/sim_ctl/sim_run.sh index 0331952..1515b09 100755 --- a/ctl/sim_ctl/sim_run.sh +++ b/ctl/sim_ctl/sim_run.sh @@ -17,10 +17,11 @@ source $LOADENVS if [[ "${modelid}" == *clm* ]]; then # Set PIO log files -if [[ -z $SLURM_JOB_ID || "$SLURM_JOB_ID" == " " ]]; then +job_id=$(sched_job_id) +if [[ -z "$job_id" || "$job_id" == " " ]]; then LOGID=$(date +%Y-%m-%d_%H.%M.%S) else - LOGID=$SLURM_JOB_ID + LOGID=$job_id fi mkdir -p logs timing/checkpoints LOGDIR=$(realpath logs) @@ -47,7 +48,11 @@ fi # parflow # Run model TIME_START=$(date +%s) echo ">>> ${modelid} started at $(date +%H:%M:%S)" -srun --multi-prog slm_multiprog_mapping.conf +if [[ "${scheduler}" == "pbs" || "${scheduler}" == "local" ]]; then + mpiexec --app ${mpmd_mapping_file} +else + srun --multi-prog ${mpmd_mapping_file} +fi TIME_END=$(date +%s) echo ">>> ${modelid} finished at $(date +%H:%M:%S)" echo ">>> ${modelid} runtime: $(date -u -d "0 $TIME_END sec - $TIME_START sec" +"%H:%M:%S")" diff --git a/ctl/utils_tsmp2.sh b/ctl/utils_tsmp2.sh index 16ca0a2..1321f83 100755 --- a/ctl/utils_tsmp2.sh +++ b/ctl/utils_tsmp2.sh @@ -7,6 +7,17 @@ # check_var_def - check if variable is defined and if not take default and printing message # logging_job_status - log information about the job into job_status.log # parse_config_file - parser to read in ini/conf-files +# +# Scheduler abstraction to faciliate different schedular +# sched_pbs_mailcode - map generic mailtype to a PBS -m code +# sched_step_opts - per-job-step submission options (name, walltime, nodes/tasks, log files) +# sched_dependency_opt - build the scheduler-specific job-dependency flag +# sched_submit - submit a job script, returns raw submission output +# sched_parse_jobid - extract the job id from submission output +# sched_job_id - current job id, read from within a running job +# sched_job_name - current job name, read from within a running job +# sched_directive_prefix - in-script batch-directive comment prefix (#SBATCH / #PBS) +# run_serial_step - run a serial helper program (job-step exclusive under slurm, direct under pbs) ## # calculate number of processors for TSMP2 application @@ -96,14 +107,168 @@ logging_job_status(){ local step="$1" if [ "$joblog" = true ] && [ "$debugmode" != true ]; then - job_id=$SLURM_JOB_ID - job_state=$(scontrol show job $job_id | grep "JobState=" | cut -d= -f2 | cut -d' ' -f1) + job_id=$(sched_job_id) + case "${scheduler:-slurm}" in + slurm) + job_state=$(scontrol show job $job_id | grep "JobState=" | cut -d= -f2 | cut -d' ' -f1) + ;; + pbs) + job_state=$(qstat -f "$job_id" 2>/dev/null | awk -F' = ' '/job_state/{print $2}') + ;; + local) + job_state="COMPLETED" + ;; + *) + job_state=$(scontrol show job $job_id | grep "JobState=" | cut -d= -f2 | cut -d' ' -f1) + ;; + esac printf "%10s %8s %3s %15s %14s %10s %10s %14s %8s\n" "${expid}" "${caseid}" "${step}" "${modelid}" \ "${dateshort}" "${job_id}" "${job_state}" "$(date '+%Y%m%d%H%M%S')" $(date -u -d "0 $timeend sec - $timestart sec" +"%H:%M:%S") \ >> ${ctl_dir}/job_status.log fi } # logging_job_status +### +# Scheduler abstraction: slurm (default), pbs, or local, selected via ${scheduler} +# 'local' runs job scripts synchronously in the foreground (no queue), for +# machines without a batch scheduler (e.g. a plain Ubuntu workstation). +### + +# map generic mailtype (NONE,BEGIN,END,FAIL,REQUEUE,ALL) to a PBS -m code +sched_pbs_mailcode() { + case "${mailtype^^}" in + BEGIN) echo "b" ;; + END) echo "e" ;; + FAIL|REQUEUE) echo "a" ;; + ALL) echo "abe" ;; + *) echo "n" ;; + esac +} # sched_pbs_mailcode + +# per-job-step submission options (job name, walltime, node/task layout, log files) +# args: jobname walltime nodes ntasks +sched_step_opts() { + local jobname="$1" walltime="$2" nodes="$3" ntasks="$4" + + case "${scheduler:-slurm}" in + slurm) + echo "--job-name=${jobname} \ + --time=${walltime} \ + --output=${log_dir}/%x_%j.out \ + --error=${log_dir}/%x_%j.err \ + --nodes=${nodes} \ + --ntasks=${ntasks}" + ;; + pbs) + local ppn=$((ntasks/nodes)) + echo "-N ${jobname} \ + -l walltime=${walltime} \ + -l nodes=${nodes}:ppn=${ppn} \ + -o ${log_dir}/ \ + -e ${log_dir}/" + ;; + local) + : # no scheduler options: sched_submit runs the job script directly + ;; + *) + echo "--job-name=${jobname} \ + --time=${walltime} \ + --output=${log_dir}/%x_%j.out \ + --error=${log_dir}/%x_%j.err \ + --nodes=${nodes} \ + --ntasks=${ntasks}" + ;; + esac +} # sched_step_opts + +# build the scheduler-specific job-dependency flag +# args: dependency payload, e.g. "afterok:123:456" (empty => no dependency) +sched_dependency_opt() { + local dep="$1" + [ -z "$dep" ] && return 0 + + case "${scheduler:-slurm}" in + slurm) echo "--dependency=${dep}" ;; + pbs) echo "-W depend=${dep}" ;; + local) : ;; # local jobs already run sequentially/blocking, no dependency needed + *) echo "--dependency=${dep}" ;; + esac +} # sched_dependency_opt + +# submit a job script; args: jobname opt_string job_script +# under 'local', runs synchronously in the foreground (blocking) and redirects +# stdout/stderr into log_dir, matching the slurm output-file naming convention +sched_submit() { + local jobname="$1" opts="$2" job_script="$3" + + case "${scheduler:-slurm}" in + slurm) + sbatch ${opts} ${job_script} 2>&1 + ;; + pbs) + qsub ${opts} ${job_script} 2>&1 + ;; + local) + local job_id=$(date +%s%N) + TSMP2_LOCAL_JOB_ID="${job_id}" TSMP2_LOCAL_JOB_NAME="${jobname}" \ + bash ${job_script} > ${log_dir}/${jobname}_${job_id}.out 2> ${log_dir}/${jobname}_${job_id}.err + echo "${job_id}" + ;; + *) + sbatch ${opts} ${job_script} 2>&1 + ;; + esac +} # sched_submit + +# extract the job id from a job's submission output +# args: submission output (e.g. captured from sched_submit) +sched_parse_jobid() { + case "${scheduler:-slurm}" in + slurm) echo "$1" | awk 'END{print $(NF)}' ;; + pbs|local) echo "$1" | tail -n1 | tr -d '[:space:]' ;; + *) echo "$1" | awk 'END{print $(NF)}' ;; + esac +} # sched_parse_jobid + +# current job id, read from within a running job +sched_job_id() { + case "${scheduler:-slurm}" in + slurm) echo "${SLURM_JOB_ID}" ;; + pbs) echo "${PBS_JOBID%%.*}" ;; + local) echo "${TSMP2_LOCAL_JOB_ID}" ;; + *) echo "${SLURM_JOB_ID}" ;; + esac +} # sched_job_id + +# current job name, read from within a running job +sched_job_name() { + case "${scheduler:-slurm}" in + slurm) echo "${SLURM_JOB_NAME}" ;; + pbs) echo "${PBS_JOBNAME}" ;; + local) echo "${TSMP2_LOCAL_JOB_NAME}" ;; + *) echo "${SLURM_JOB_NAME}" ;; + esac +} # sched_job_name + +# in-script batch-directive comment prefix, used when generating job scripts in debugmode +sched_directive_prefix() { + case "${scheduler:-slurm}" in + slurm) echo "#SBATCH" ;; + pbs) echo "#PBS" ;; + local) echo "#" ;; + *) echo "#SBATCH" ;; + esac +} # sched_directive_prefix + +# run a serial helper program: an exclusive job-step under slurm, directly under pbs/local +run_serial_step() { + case "${scheduler:-slurm}" in + slurm) srun --exclusive -n 1 "$@" ;; + pbs|local) "$@" ;; + *) srun --exclusive -n 1 "$@" ;; + esac +} # run_serial_step + # input 1: filename of conf-file input 2: section (optional) parse_config_file() { diff --git a/ctl/vis_ctl/vis.job b/ctl/vis_ctl/vis.job index c4ecfee..f24be03 100755 --- a/ctl/vis_ctl/vis.job +++ b/ctl/vis_ctl/vis.job @@ -3,6 +3,7 @@ ### # Import functions ### +source ${ctl_dir}/utils_tsmp2.sh source ${ctl_dir}/vis_ctl/vis_config.sh source ${ctl_dir}/vis_ctl/vis_run.sh source ${ctl_dir}/vis_ctl/vis_cleanup.sh diff --git a/doc/user_guide/usage_wfe/running.md b/doc/user_guide/usage_wfe/running.md index e95e475..7528379 100644 --- a/doc/user_guide/usage_wfe/running.md +++ b/doc/user_guide/usage_wfe/running.md @@ -39,6 +39,12 @@ If the simulation terminates prematurely (e.g., due to a crash) before the clean ### Main Settings +#### scheduler + +Job scheduler to submit workflow jobs to compute. Options: `slurm` (default), `pbs` (OpenPBS/Torque), or `local`. Selects the submission command (`sbatch`/`qsub`/none), resource-request syntax, dependency handling, and the MPMD launch mechanism used for the coupled simulation step (Slurm `srun --multi-prog` vs. an Open MPI `mpiexec --app` mapping file, used by both `pbs` and `local`). + +`local` is for machines without a batch scheduler (e.g. a plain Ubuntu workstation): each workflow stage runs synchronously in the foreground (no queue, no wallclock/partition/account/mail settings), with stdout/stderr redirected into `ctl/logs/`. Requires a working local MPI installation (`mpiexec`) for the simulation step, and enough local cores to cover `npnode`. + #### MODEL_ID Specifies which model components to use. Options include but are not @@ -139,8 +145,9 @@ only executed once the previous is done. #### mailtype -Email notification trigger for SLURM jobs. Options: `NONE`, `BEGIN`, -`END`, `FAIL`, `REQUEUE`, `ALL`. +Email notification trigger for scheduler jobs. Options: `NONE`, `BEGIN`, +`END`, `FAIL`, `REQUEUE`, `ALL`. Mapped to the equivalent PBS `-m` code +when `scheduler=pbs`. #### mailaddress @@ -159,7 +166,8 @@ Number of cores per node. Leave empty to use machine defaults. #### partition -Compute partition to use. Leave empty to use machine defaults. +Compute partition (`slurm`) or queue (`pbs`) to use. Leave empty to use +machine defaults. #### account