diff --git a/scripts/run.sh b/scripts/run.sh index a56fb6a..68b382c 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -2,14 +2,31 @@ RECOVERY_DONE_FILE="/var/pv/"$PITR_UNIX_TIME"_recovery.done" PITR_RS=${PITR_REPLICATION_STRATEGY:-none} STOP=false -# don't restart postgres on SIGTERM (eg, pod deleted) +# don't restart postgres on SIGTERM (eg, pod deleted), and stop supervising once +# the signal arrives, so this script (the container's main process) EXITS. +# +# The trap alone is not enough: it only tells the loop below not to start postgres +# again. With `while true` the loop kept spinning forever after that, the container +# never exited on its own, and kubelet had no choice but to wait out the whole +# terminationGracePeriodSeconds and SIGKILL. Two consequences, both observed live: +# pod deletions blocked for the entire grace period (5 minutes on a DB that sets +# 300), and on the default 30s grace a postgres shutdown checkpoint slower than +# that got SIGKILLed midway, leaving a data directory that needs crash recovery. +# +# STOP is only ever set by a signal delivered to THIS bash process, which in +# practice means kubelet stopping the container. The coordinator stops postgres +# with `pg_ctl stop` exec'd into the container (and kills nothing else by name or +# process group), so its shutdowns do not reach this process: the loop keeps +# retrying and re-runs the role script when the coordinator writes the next one, +# exactly as before. Verified live: a single container instance served 17 role +# script starts with no restart. # ref: https://opensource.com/article/20/6/bash-trap trap \ "{ STOP=true; }" \ SIGINT SIGTERM EXIT if [[ "$PITR_RESTORE" == "true" ]]; then - while true; do + while [[ "$STOP" = false ]]; do sleep 2 echo "Point In Time Recovery In Progress. Waiting for $RECOVERY_DONE_FILE file" if [[ -e "$RECOVERY_DONE_FILE" ]]; then @@ -22,7 +39,7 @@ fi #going to change this with the check of process id rm -f "$PGDATA"/postmaster.pid echo "waiting for the role to be decided ..." -while true; do +while [[ "$STOP" = false ]]; do # Robust /var/pv mount availability check before any destructive operation or basebackup if [[ -e /var/pv/BOOTSTRAP_INITIALIZATION_STARTED ]]; then rm /var/pv/BOOTSTRAP_INITIALIZATION_STARTED