From 42641ebfd7981e70ca863e81c40b70f26bc0384c Mon Sep 17 00:00:00 2001 From: vsutra-admin Date: Sun, 21 Jun 2026 07:15:57 +0530 Subject: [PATCH] feat: auto-rollback when requested release is older than installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unifies deploy and rollback into a single operator flow — operator picks a release in site.yml/deploy.yml (or in the Jenkins deploy dropdown), and the playbook routes to rollback automatically if the requested version is older than what is installed. No separate rollback job needed. Refactor: - roles/openspecimen/tasks/rollback.yml (new) — extracted from rollback.yml's play body. Supports three backup-selection modes: * backup_timestamp — explicit timestamp * target_version — find backup whose .release matches this string (used by the auto-rollback path) * (neither) — most recent backup by mtime - roles/openspecimen/tasks/direction.yml (new) — detects deploy vs rollback direction from openspecimen_release vs the on-target .release marker using sort -V (handles RC12 > RC8 correctly). On downgrade: dispatches to tasks_from: rollback with target_version, then ends play. - rollback.yml (refactor) — thin wrapper that just includes the role tasks_from. Operator CLI invocation unchanged. Wiring: - site.yml / deploy.yml — direction.yml runs as the FIRST pre_task. On rollback, the subsequent preflight + roles are skipped via end_play. Schema-incompatible downgrade safeguard from PR #25 still applies — it lives in the rollback task list, so both invocation paths get it. Docs: - docs/DEPLOY-UPGRADE.md — 'How version detection works' updated for auto-rollback; Rollback section now distinguishes path A (automatic via deploy job) from path B (explicit rollback.yml invocation), with target_version usage documented. --- deploy.yml | 8 + docs/DEPLOY-UPGRADE.md | 65 +++- roles/openspecimen/tasks/direction.yml | 67 ++++ roles/openspecimen/tasks/rollback.yml | 409 +++++++++++++++++++++++++ rollback.yml | 404 ++---------------------- site.yml | 9 + 6 files changed, 563 insertions(+), 399 deletions(-) create mode 100644 roles/openspecimen/tasks/direction.yml create mode 100644 roles/openspecimen/tasks/rollback.yml diff --git a/deploy.yml b/deploy.yml index c5eb6d662d..c8eba16ad8 100644 --- a/deploy.yml +++ b/deploy.yml @@ -28,6 +28,14 @@ openspecimen_install_type: "{{ openspecimen_install_type | default('upgrade') }}" pre_tasks: + # Detect deploy vs rollback direction. On downgrade, the rollback flow + # is invoked automatically and the play ends — the openspecimen role + # below does not run. + - name: Detect direction (deploy or rollback) + ansible.builtin.include_role: + name: openspecimen + tasks_from: direction + # Verify the release zip and every paid/customer plugin zip exists on the # control node before the openspecimen role runs. Fails fast if missing. - name: Pre-flight artifact check diff --git a/docs/DEPLOY-UPGRADE.md b/docs/DEPLOY-UPGRADE.md index f56618296c..ebfae9945c 100644 --- a/docs/DEPLOY-UPGRADE.md +++ b/docs/DEPLOY-UPGRADE.md @@ -6,16 +6,23 @@ Day-1 fresh install, Day-2 upgrade, and rollback procedures for OpenSpecimen. ## How version detection works -The roles use the marker file `/usr/local/openspecimen/.release` on the target -node to determine what is currently installed: +The playbooks use the marker file `/usr/local/openspecimen/.release` on the +target node to determine what is currently installed, then compare against the +requested `openspecimen_release` using natural version sort (`sort -V`): -- **File absent** → fresh install (all five roles run in order) -- **File present, same version** → no-op (idempotent run) -- **File present, lower version** → upgrade (backup → deploy → restart) -- **File present, higher version** → downgrade — **blocked** with a clear error +- **Marker absent** → fresh install (all five roles run in order) +- **Marker present, same version** → re-deploy (idempotent run) +- **Marker present, requested > installed** → upgrade (backup → deploy → restart) +- **Marker present, requested < installed** → automatic rollback (see "Rollback" below) — + the requested version's backup is restored and the play ends. No separate job needed. -`openspecimen_release` is always supplied at run time via `-e openspecimen_release=`. -It is **not** stored in inventory `group_vars`. +`openspecimen_release` is always supplied at run time via +`-e openspecimen_release=`. It is **not** stored in inventory `group_vars`. + +Direction detection runs as the **first** pre_task in both `site.yml` and +`deploy.yml` — `roles/openspecimen/tasks/direction.yml`. When a downgrade is +detected, the play dispatches to `tasks_from: rollback` (target_version = +requested release) and ends; the remaining roles never run. --- @@ -190,32 +197,56 @@ Each timestamped directory holds a complete snapshot: ## Day-2: Rollback -Use the `rollback.yml` playbook. It restores the WAR, all three plugin tiers, -the MySQL connector JAR (when backed up), and the `.release` marker from a -timestamped backup, then waits for the health check. Target: ≤ 5 minutes. +Rollback has two paths — both backed by the same `roles/openspecimen/tasks/rollback.yml`: + +### A. Automatic (via deploy job — recommended) -### Roll back to the most recent backup (default) +Just pick a lower release in `site.yml` / `deploy.yml` (or in the Jenkins +deploy job). Direction detection notices the requested version is older than +what's installed, finds the backup whose `.release` matches the request, +runs the rollback, and ends the play. **No separate rollback job or playbook +invocation required.** ```bash -ansible-playbook -i inventory/customers// rollback.yml \ +# Installed: openspecimen_v12.2.RC12, want to go back to RC8 +ansible-playbook -i inventory/customers// site.yml \ + -e openspecimen_release=openspecimen_v12.2.RC8 \ -e @secrets/.yml --vault-password-file .vault-pass ``` -### Roll back to a specific timestamped backup +If no backup of the requested version exists, the play fails with a list of +available backups and the operator can either pick a version that does have +one or override with `-e allow_downgrade=true`. + +### B. Explicit — direct rollback.yml invocation -List available backups: +Use this when you want to roll back to a specific backup directory (or the +most recent one) rather than a specific version. + +**Most recent backup (default):** + +```bash +ansible-playbook -i inventory/customers// rollback.yml \ + -e @secrets/.yml --vault-password-file .vault-pass +``` + +**Specific timestamped backup** (list first, then pass): ```bash ansible -i inventory/customers// openspecimen -b \ -a "ls /usr/local/openspecimen/backup/" # example output: 17052026_093247 18052026_104530 19052026_110042 + +ansible-playbook -i inventory/customers// rollback.yml \ + -e backup_timestamp=17052026_093247 \ + -e @secrets/.yml --vault-password-file .vault-pass ``` -Then pass the chosen timestamp: +**Specific OpenSpecimen version** (let the playbook locate the matching backup): ```bash ansible-playbook -i inventory/customers// rollback.yml \ - -e backup_timestamp=17052026_093247 \ + -e target_version=openspecimen_v12.1.RC8 \ -e @secrets/.yml --vault-password-file .vault-pass ``` diff --git a/roles/openspecimen/tasks/direction.yml b/roles/openspecimen/tasks/direction.yml new file mode 100644 index 0000000000..201661df08 --- /dev/null +++ b/roles/openspecimen/tasks/direction.yml @@ -0,0 +1,67 @@ +--- +# Detect deploy/rollback direction from openspecimen_release vs the on-target +# .release marker. When a downgrade is detected, the rollback flow is invoked +# automatically (using openspecimen_release as the target_version), then the +# play ends — site.yml's other roles do not run. +# +# Called as the FIRST pre_task from site.yml and deploy.yml. + +- name: Validate openspecimen_release is set (required for direction detection) + ansible.builtin.fail: + msg: | + ✗ openspecimen_release is not set. + + What to do: + Pass at run time: -e openspecimen_release=openspecimen_v12.2.RC12 + From Jenkins: select RELEASE_FILE in the deploy job. + when: not (openspecimen_release | default('') | trim) + +- name: Read installed version marker on target + ansible.builtin.slurp: + src: "{{ openspecimen_release_marker }}" + register: _direction_marker + failed_when: false + changed_when: false + +- name: Set installed release fact + ansible.builtin.set_fact: + _installed_release: >- + {{ (_direction_marker.content | b64decode | trim) + if _direction_marker.content is defined else '' }} + +# Use sort -V (natural version sort) so RC12 > RC8 sorts correctly. +- name: Detect direction (deploy / rollback) + ansible.builtin.shell: | + INSTALLED="{{ _installed_release }}" + REQUESTED="{{ openspecimen_release }}" + if [ -z "$INSTALLED" ] || [ "$INSTALLED" = "$REQUESTED" ]; then + echo deploy + elif [ "$(printf '%s\n%s\n' "$INSTALLED" "$REQUESTED" | sort -V | head -1)" = "$REQUESTED" ]; then + echo rollback + else + echo deploy + fi + register: _direction + changed_when: false + +- name: Announce detected direction + ansible.builtin.debug: + msg: >- + {% if _direction.stdout == 'rollback' %} + ⤺ Downgrade detected: installed {{ _installed_release }} → requested {{ openspecimen_release }}. + Switching to ROLLBACK flow — will restore from the backup whose .release matches {{ openspecimen_release }}. + {% elif _installed_release %} + → Deploy: installed {{ _installed_release }} → requested {{ openspecimen_release }}. + {% else %} + → Fresh install: requested {{ openspecimen_release }}. + {% endif %} + +- name: Run rollback flow when downgrade detected + ansible.builtin.include_tasks: rollback.yml + vars: + target_version: "{{ openspecimen_release }}" + when: _direction.stdout == 'rollback' + +- name: End play after rollback (skip deploy roles) + ansible.builtin.meta: end_play + when: _direction.stdout == 'rollback' diff --git a/roles/openspecimen/tasks/rollback.yml b/roles/openspecimen/tasks/rollback.yml new file mode 100644 index 0000000000..3dd4e020c0 --- /dev/null +++ b/roles/openspecimen/tasks/rollback.yml @@ -0,0 +1,409 @@ +--- +# Rollback flow — included from two places: +# +# 1. Top-level rollback.yml (operator runs explicitly) +# 2. site.yml / deploy.yml pre_task — when the requested release is older +# than the installed release (downgrade direction detected) +# +# Backup selection (in order of precedence): +# 1. backup_timestamp : restore from this specific timestamped directory +# 2. target_version : restore from the backup whose .release matches this +# (used by the auto-rollback path — pass the requested +# openspecimen_release as target_version) +# 3. (neither) : restore from the most recent backup (mtime) +# +# Safety: halts before stopping the service if Liquibase has applied changesets +# after the backup was created. Override with allow_downgrade=true. + +# ── 1. Resolve which backup to restore ─────────────────────────────────────── + +- name: List timestamped backup directories on target + ansible.builtin.find: + paths: "{{ openspecimen_backup_dir }}" + file_type: directory + excludes: "config-changes" + recurse: false + register: _backup_dirs + +- name: Fail with operator guidance if no backups exist + ansible.builtin.fail: + msg: | + ✗ No timestamped backups found under {{ openspecimen_backup_dir }}. + + Customer: {{ inventory_hostname }} + + What likely happened: + - This is the first deploy and no upgrade has occurred yet, OR + - All backups were pruned by openspecimen_backup_retention, OR + - The target node was rebuilt and previous backups are lost. + + What to check: + ls -la {{ openspecimen_backup_dir }}/ + + How to fix: + - There is nothing to roll back to. To deploy an older version, + run deploy.yml with -e openspecimen_release= and + -e allow_downgrade=true, and place the older release zip in + openspecimen_builds_dir. + when: _backup_dirs.files | length == 0 + +# Compute most-recent backup (used when neither backup_timestamp nor +# target_version is provided). +- name: Compute most-recent backup path + ansible.builtin.set_fact: + _latest_backup_path: "{{ (_backup_dirs.files | sort(attribute='mtime', reverse=true) | first).path }}" + when: _backup_dirs.files | length > 0 + +# Find the backup whose .release marker matches target_version (used by the +# auto-rollback path triggered from site.yml when a downgrade is detected). +- name: Find backup directory matching target_version + ansible.builtin.shell: | + set -eo pipefail + for d in "{{ openspecimen_backup_dir }}"/*/; do + [ -d "$d" ] || continue + [ -f "${d}.release" ] || continue + v=$(tr -d '[:space:]' < "${d}.release") + if [ "$v" = "{{ target_version }}" ]; then + echo "${d%/}" + exit 0 + fi + done + exit 1 + args: + executable: /bin/bash + register: _target_version_match + failed_when: false + changed_when: false + when: + - target_version is defined + - target_version | default('') | trim + - not (backup_timestamp is defined and backup_timestamp | default('') | trim) + +- name: Resolve backup directory to restore from + ansible.builtin.set_fact: + _restore_path: >- + {%- if backup_timestamp is defined and backup_timestamp | trim -%} + {{ openspecimen_backup_dir.rstrip('/') }}/{{ backup_timestamp | trim }} + {%- elif target_version is defined and target_version | default('') | trim -%} + {%- if _target_version_match.rc | default(1) == 0 -%} + {{ _target_version_match.stdout | trim }} + {%- else -%} + NO_MATCH + {%- endif -%} + {%- else -%} + {{ _latest_backup_path }} + {%- endif -%} + +- name: Fail when target_version has no matching backup + ansible.builtin.fail: + msg: | + ✗ No backup found for version {{ target_version }} on {{ inventory_hostname }}. + + Available backups (most recent first): + {% for f in (_backup_dirs.files | sort(attribute='mtime', reverse=true)) %} + - {{ f.path | basename }} (mtime {{ '%Y-%m-%d %H:%M:%S' | strftime(f.mtime) }}) + {% endfor %} + + How to fix: + - Pick a version that has a backup directory above, OR + - Restore the database from an RDS snapshot taken before that version's + rollout and re-run with -e allow_downgrade=true on deploy.yml. + when: _restore_path == 'NO_MATCH' + +- name: Verify the resolved backup exists + ansible.builtin.stat: + path: "{{ _restore_path }}" + register: _restore_stat + +- name: Fail with operator guidance if the requested timestamp is missing + ansible.builtin.fail: + msg: | + ✗ Backup directory not found: {{ _restore_path }} + + Customer: {{ inventory_hostname }} + Requested: {{ backup_timestamp | default('(latest)') }} + + Available backups (most recent first): + {% for f in (_backup_dirs.files | sort(attribute='mtime', reverse=true)) %} + - {{ f.path | basename }} (mtime {{ '%Y-%m-%d %H:%M:%S' | strftime(f.mtime) }}) + {% endfor %} + + How to fix: + ansible-playbook -i inventory/customers/{{ inventory_hostname }}/ rollback.yml \ + -e backup_timestamp= + when: not _restore_stat.stat.exists + +# ── 2. Verify the backup contains a WAR (sanity check) ─────────────────────── + +- name: Verify the backup contains openspecimen.war + ansible.builtin.stat: + path: "{{ _restore_path }}/openspecimen.war" + register: _war_stat + +- name: Fail if the backup is incomplete (no WAR) + ansible.builtin.fail: + msg: | + ✗ Backup is incomplete: {{ _restore_path }}/openspecimen.war is missing. + + This backup directory may have been partially deleted, or was created + by an older version of the playbook that did not back up the WAR. + Pick a different timestamp from the list above. + when: not _war_stat.stat.exists + +- name: Show planned rollback + ansible.builtin.debug: + msg: + - "Restoring from: {{ _restore_path }}" + - "Backup mtime : {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }}" + - "WAR size : {{ (_war_stat.stat.size / 1024 / 1024) | round(1) }} MB" + +# ── 2b. Schema-incompatible downgrade detection ────────────────────────────── + +- name: Read live release marker (for diagnostics) + ansible.builtin.slurp: + src: "{{ openspecimen_release_marker }}" + register: _live_marker_slurp + failed_when: false + changed_when: false + +- name: Read backup release marker (for diagnostics) + ansible.builtin.slurp: + src: "{{ _restore_path }}/.release" + register: _backup_marker_slurp + failed_when: false + changed_when: false + +- name: Count Liquibase migrations applied after the backup + ansible.builtin.shell: | + set -eo pipefail + mysql -u {{ mysql_db_user }} -p'{{ mysql_db_password }}' \ + -h {{ mysql_db_host }} {{ mysql_db_name }} \ + -sNe "SELECT COUNT(*) FROM DATABASECHANGELOG + WHERE DATEEXECUTED > FROM_UNIXTIME({{ _restore_stat.stat.mtime | int }});" + args: + executable: /bin/bash + register: _changelog_count + changed_when: false + failed_when: false + no_log: true + when: + - db_type | default('mysql') == 'mysql' + - mysql_db_password is defined + +- name: Halt rollback if schema has moved forward since backup + ansible.builtin.fail: + msg: | + ✗ Schema-incompatible downgrade detected. + + Customer: {{ inventory_hostname }} + Live release: {{ (_live_marker_slurp.content | b64decode | trim) if _live_marker_slurp.content is defined else '(no marker)' }} + Backup release: {{ (_backup_marker_slurp.content | b64decode | trim) if _backup_marker_slurp.content is defined else '(no marker)' }} + Backup mtime: {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }} + New migrations: {{ _changelog_count.stdout | trim }} row(s) in DATABASECHANGELOG since backup + + What likely happened: + OpenSpecimen applied Liquibase migrations after this backup was created. + Rolling back the WAR without rolling back the schema would leave the older + application running against a newer schema — risk of runtime errors, missing + columns, or refused startup. + + What to check (list the offending changesets): + mysql -u {{ mysql_db_user }} -p -h {{ mysql_db_host }} {{ mysql_db_name }} \ + -e "SELECT ID, AUTHOR, FILENAME, DATEEXECUTED FROM DATABASECHANGELOG \ + WHERE DATEEXECUTED > FROM_UNIXTIME({{ _restore_stat.stat.mtime | int }}) \ + ORDER BY DATEEXECUTED ASC;" + + How to fix (pick one): + 1. Use a MORE RECENT backup that postdates the new migrations. + 2. Restore the database from an RDS snapshot taken before + {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }}, then re-run. + 3. Override (use only after manual schema fix or confirmed-safe downgrade): + ansible-playbook ... -e allow_downgrade=true + — in Jenkins: tick the ALLOW_DOWNGRADE checkbox on the deploy job. + when: + - not (allow_downgrade | default(false) | bool) + - _changelog_count is defined + - _changelog_count.stdout is defined + - (_changelog_count.stdout | trim | int) > 0 + +- name: Warn that schema-downgrade detection was bypassed + ansible.builtin.debug: + msg: >- + ⚠ allow_downgrade=true — skipping schema-downgrade detection. + Proceeding with the rollback. Make sure the database schema is + compatible with the target WAR before continuing. + when: + - allow_downgrade | default(false) | bool + - _changelog_count is defined + - (_changelog_count.stdout | default('0') | trim | int) > 0 + +# ── 3. Stop the service ────────────────────────────────────────────────────── + +- name: Stop openspecimen service + ansible.builtin.systemd: + name: openspecimen + state: stopped + +# ── 4. Restore WAR (replace the live one) ──────────────────────────────────── + +- name: Remove current WAR + expanded webapp directory + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ tomcat_home }}/webapps/openspecimen.war" + - "{{ tomcat_home }}/webapps/openspecimen" + +- name: Restore openspecimen.war from backup + ansible.builtin.copy: + src: "{{ _restore_path }}/openspecimen.war" + dest: "{{ tomcat_home }}/webapps/openspecimen.war" + remote_src: true + owner: "{{ tomcat_user }}" + group: "{{ tomcat_user }}" + mode: '0644' + +# ── 5. Restore plugins — all three tiers ───────────────────────────────────── + +- name: Probe backup plugin subdirectories + ansible.builtin.stat: + path: "{{ _restore_path }}/plugins/{{ item }}" + loop: [default, paid, zustomer] + register: _backup_plugin_stat + +- name: Wipe current plugin tier (only when backup has it) + ansible.builtin.shell: | + rm -f {{ openspecimen_plugin_dir }}/{{ item.item }}/*.jar + loop: "{{ _backup_plugin_stat.results }}" + when: item.stat.exists + changed_when: true + +- name: Restore plugin JARs from backup + ansible.builtin.shell: | + if compgen -G "{{ _restore_path }}/plugins/{{ item.item }}/*.jar" > /dev/null; then + cp {{ _restore_path }}/plugins/{{ item.item }}/*.jar {{ openspecimen_plugin_dir }}/{{ item.item }}/ + chown {{ tomcat_user }}:{{ tomcat_user }} {{ openspecimen_plugin_dir }}/{{ item.item }}/*.jar + chmod 0644 {{ openspecimen_plugin_dir }}/{{ item.item }}/*.jar + echo "restored" + else + echo "empty" + fi + args: + executable: /bin/bash + loop: "{{ _backup_plugin_stat.results }}" + when: item.stat.exists + register: _restore_plugins + changed_when: _restore_plugins.stdout == 'restored' + +# ── 6. Restore MySQL connector JAR if backed up ────────────────────────────── + +- name: Probe backup lib directory + ansible.builtin.stat: + path: "{{ _restore_path }}/lib" + register: _backup_lib_stat + +- name: Restore MySQL connector JAR from backup lib/ + ansible.builtin.shell: | + if compgen -G "{{ _restore_path }}/lib/mysql-connector*.jar" > /dev/null; then + rm -f {{ tomcat_home }}/lib/mysql-connector*.jar + cp {{ _restore_path }}/lib/mysql-connector*.jar {{ tomcat_home }}/lib/ + chown {{ tomcat_user }}:{{ tomcat_user }} {{ tomcat_home }}/lib/mysql-connector*.jar + chmod 0644 {{ tomcat_home }}/lib/mysql-connector*.jar + echo "restored" + else + echo "skipped" + fi + args: + executable: /bin/bash + when: _backup_lib_stat.stat.exists + register: _restore_connector + changed_when: _restore_connector.stdout == 'restored' + +# ── 7. Restore config files from /config/ ──────────────────────────── + +- name: Probe backup config files + ansible.builtin.stat: + path: "{{ _restore_path }}/config/{{ item.name }}" + loop: + - { name: openspecimen.properties, dest: "{{ tomcat_home }}/conf/openspecimen.properties", mode: '0640' } + - { name: setenv.sh, dest: "{{ tomcat_home }}/bin/setenv.sh", mode: '0755' } + - { name: context.xml, dest: "{{ tomcat_home }}/conf/context.xml", mode: '0640' } + loop_control: + label: "{{ item.name }}" + register: _backup_config_stat + changed_when: false + +- name: Restore config files from backup + ansible.builtin.copy: + src: "{{ _restore_path }}/config/{{ item.item.name }}" + dest: "{{ item.item.dest }}" + remote_src: true + owner: "{{ tomcat_user }}" + group: "{{ tomcat_user }}" + mode: "{{ item.item.mode }}" + loop: "{{ _backup_config_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: item.stat.exists + +- name: Warn when backup has no config snapshot + ansible.builtin.debug: + msg: >- + NOTE: backup {{ _restore_path | basename }} has no config/ subdirectory. + The live openspecimen.properties / setenv.sh / context.xml are left in + place. If config was changed between this backup and now, re-run + site.yml to re-template them from inventory. + when: not (_backup_config_stat.results | map(attribute='stat.exists') | select | list | length > 0) + +# ── 8. Update the release marker to reflect the rolled-back version ────────── + +- name: Probe backed-up release marker (if present) + ansible.builtin.stat: + path: "{{ _restore_path }}/.release" + register: _backup_release_marker + +- name: Restore release marker from backup + ansible.builtin.copy: + src: "{{ _restore_path }}/.release" + dest: "{{ openspecimen_release_marker }}" + remote_src: true + owner: "{{ tomcat_user }}" + group: "{{ tomcat_user }}" + mode: '0644' + when: _backup_release_marker.stat.exists + +- name: Warn when no release marker is in the backup + ansible.builtin.debug: + msg: >- + NOTE: backup {{ _restore_path | basename }} has no .release marker. + The on-disk marker still shows the previous (rolled-forward) version + and may incorrectly block the next deploy as a downgrade. If that + happens, remove {{ openspecimen_release_marker }} manually before + re-running deploy.yml. + when: not _backup_release_marker.stat.exists + +# ── 9. Start service and wait for readiness ────────────────────────────────── + +- name: Start openspecimen service + ansible.builtin.systemd: + name: openspecimen + state: started + daemon_reload: true + +- name: Wait for the app to respond + ansible.builtin.shell: > + curl -sf -o /dev/null -w "%{http_code}" + http://localhost:{{ openspecimen_port }}/openspecimen/rest/ng/config-settings/app-props + | grep -qE '^(200|401)$' + register: _rollback_health + until: _rollback_health.rc == 0 + retries: 30 + delay: 10 + changed_when: false + +- name: Rollback complete + ansible.builtin.debug: + msg: >- + ✓ Rolled back to {{ _restore_path | basename }} + (mtime {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }}) + on {{ inventory_hostname }}. diff --git a/rollback.yml b/rollback.yml index a122d59cca..4fe629d74e 100644 --- a/rollback.yml +++ b/rollback.yml @@ -1,27 +1,20 @@ --- # Day-2: Roll back OpenSpecimen to a previous timestamped backup. # -# Default behaviour: restores the most recent backup (the one created by the -# last deploy/upgrade). Optionally pass -e backup_timestamp= -# to roll back to a specific older backup. +# This is a thin wrapper around roles/openspecimen/tasks/rollback.yml so that +# the same tasks are usable as both: +# 1. A standalone playbook (operator CLI — this file), and +# 2. An included task list (site.yml / deploy.yml when a downgrade is detected +# automatically — see roles/openspecimen/tasks/direction.yml). # -# What it restores: -# - openspecimen.war -# - plugins/{default,paid,zustomer}/*.jar -# - lib/mysql-connector-*.jar (when present in the backup) -# - config/openspecimen.properties (when present in the backup) -# - config/setenv.sh (JVM heap — when present in the backup) -# - config/context.xml (JDBC pool — when present in the backup) -# - .release marker (so the next deploy doesn't misfire) +# Backup selection (in order of precedence): +# 1. backup_timestamp : restore from this specific timestamped directory +# 2. target_version : restore from the backup whose .release matches this +# 3. (neither) : restore from the most recent backup (by mtime) # -# Safety: halts the rollback when Liquibase has applied changesets after the -# backup was created (schema-incompatible downgrade). Override only if you -# have verified the schema is safe: -e allow_downgrade=true -# -# What it does NOT restore: -# - Database schema — Liquibase rollback is not modelled. If a schema -# downgrade is required, restore the DB from an RDS snapshot first, -# then run rollback.yml. +# Safety: halts before stopping the service if Liquibase has applied changesets +# after the backup was created (schema-incompatible downgrade). Override with +# -e allow_downgrade=true. # # Usage: # # Roll back to the most recent backup (default) @@ -33,12 +26,17 @@ # -e backup_timestamp=15062026_103247 \ # -e @secrets/.yml --vault-password-file .vault-pass # +# # Roll back to whichever backup matches a specific OpenSpecimen version +# ansible-playbook -i inventory/customers// rollback.yml \ +# -e target_version=openspecimen_v12.1.RC8 \ +# -e @secrets/.yml --vault-password-file .vault-pass +# # # Override schema-downgrade halt (use only when you're sure) # ansible-playbook -i inventory/customers// rollback.yml \ # -e allow_downgrade=true \ # -e @secrets/.yml --vault-password-file .vault-pass # -# # Dry run (no service stop, no file moves, just confirms the backup exists) +# # Dry run (no service stop, no file moves) # ansible-playbook -i inventory/customers// rollback.yml --check # # Target: ≤ 5 minutes total. @@ -47,366 +45,8 @@ hosts: openspecimen become: true - pre_tasks: - - name: Show rollback diagnostics - ansible.builtin.debug: - msg: - - "=== Rollback ===" - - "host : {{ inventory_hostname }}" - - "backup_dir : {{ openspecimen_backup_dir }}" - - "requested target : {{ backup_timestamp | default('(latest)') }}" - when: openspecimen_debug | default(false) | bool - tasks: - - # ── 1. Resolve which backup to restore ─────────────────────────────────── - - - name: List timestamped backup directories on target - ansible.builtin.find: - paths: "{{ openspecimen_backup_dir }}" - file_type: directory - excludes: "config-changes" - recurse: false - register: _backup_dirs - - - name: Fail with operator guidance if no backups exist - ansible.builtin.fail: - msg: | - ✗ No timestamped backups found under {{ openspecimen_backup_dir }}. - - Customer: {{ inventory_hostname }} - - What likely happened: - - This is the first deploy and no upgrade has occurred yet, OR - - All backups were pruned by openspecimen_backup_retention, OR - - The target node was rebuilt and previous backups are lost. - - What to check: - ls -la {{ openspecimen_backup_dir }}/ - - How to fix: - - There is nothing to roll back to. To deploy an older version, - run deploy.yml with -e openspecimen_release= - and the older release zip in openspecimen_builds_dir. - when: _backup_dirs.files | length == 0 - - - name: Compute most-recent backup path (used when backup_timestamp is unset) - ansible.builtin.set_fact: - _latest_backup_path: "{{ (_backup_dirs.files | sort(attribute='mtime', reverse=true) | first).path }}" - when: _backup_dirs.files | length > 0 - - - name: Resolve backup directory to restore from - ansible.builtin.set_fact: - _restore_path: >- - {{ - (openspecimen_backup_dir.rstrip('/') + '/' + backup_timestamp) - if (backup_timestamp is defined and backup_timestamp | trim) - else _latest_backup_path - }} - - - name: Verify the requested backup exists - ansible.builtin.stat: - path: "{{ _restore_path }}" - register: _restore_stat - - - name: Fail with operator guidance if the requested timestamp is missing - ansible.builtin.fail: - msg: | - ✗ Backup directory not found: {{ _restore_path }} - - Customer: {{ inventory_hostname }} - Requested: {{ backup_timestamp | default('(latest)') }} - - Available backups (most recent first): - {% for f in (_backup_dirs.files | sort(attribute='mtime', reverse=true)) %} - - {{ f.path | basename }} (mtime {{ '%Y-%m-%d %H:%M:%S' | strftime(f.mtime) }}) - {% endfor %} - - How to fix: - ansible-playbook -i inventory/customers/{{ inventory_hostname }}/ rollback.yml \ - -e backup_timestamp= - when: not _restore_stat.stat.exists - - # ── 2. Verify the backup contains a WAR (sanity check) ─────────────────── - - - name: Verify the backup contains openspecimen.war - ansible.builtin.stat: - path: "{{ _restore_path }}/openspecimen.war" - register: _war_stat - - - name: Fail if the backup is incomplete (no WAR) - ansible.builtin.fail: - msg: | - ✗ Backup is incomplete: {{ _restore_path }}/openspecimen.war is missing. - - This backup directory may have been partially deleted, or was created - by an older version of the playbook that did not back up the WAR. - Pick a different timestamp from the list above. - when: not _war_stat.stat.exists - - - name: Show planned rollback - ansible.builtin.debug: - msg: - - "Restoring from: {{ _restore_path }}" - - "Backup mtime : {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }}" - - "WAR size : {{ (_war_stat.stat.size / 1024 / 1024) | round(1) }} MB" - - # ── 2b. Schema-incompatible downgrade detection ────────────────────────── - # If any Liquibase changesets have been applied AFTER the backup was created, - # the live schema has moved forward — rolling the WAR back to an older version - # may produce runtime errors or refuse to start. Halt unless explicitly - # overridden with -e allow_downgrade=true. - - - name: Read live release marker (for diagnostics) - ansible.builtin.slurp: - src: "{{ openspecimen_release_marker }}" - register: _live_marker_slurp - failed_when: false - changed_when: false - - - name: Read backup release marker (for diagnostics) - ansible.builtin.slurp: - src: "{{ _restore_path }}/.release" - register: _backup_marker_slurp - failed_when: false - changed_when: false - - - name: Count Liquibase migrations applied after the backup - ansible.builtin.shell: | - set -eo pipefail - mysql -u {{ mysql_db_user }} -p'{{ mysql_db_password }}' \ - -h {{ mysql_db_host }} {{ mysql_db_name }} \ - -sNe "SELECT COUNT(*) FROM DATABASECHANGELOG - WHERE DATEEXECUTED > FROM_UNIXTIME({{ _restore_stat.stat.mtime | int }});" - args: - executable: /bin/bash - register: _changelog_count - changed_when: false - failed_when: false - no_log: true - when: - - db_type | default('mysql') == 'mysql' - - mysql_db_password is defined - - - name: Halt rollback if schema has moved forward since backup - ansible.builtin.fail: - msg: | - ✗ Schema-incompatible downgrade detected. - - Customer: {{ inventory_hostname }} - Live release: {{ (_live_marker_slurp.content | b64decode | trim) if _live_marker_slurp.content is defined else '(no marker)' }} - Backup release: {{ (_backup_marker_slurp.content | b64decode | trim) if _backup_marker_slurp.content is defined else '(no marker)' }} - Backup mtime: {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }} - New migrations: {{ _changelog_count.stdout | trim }} row(s) in DATABASECHANGELOG since backup - - What likely happened: - OpenSpecimen applied Liquibase migrations after this backup was created. - Rolling back the WAR without rolling back the schema would leave the older - application running against a newer schema — risk of runtime errors, missing - columns, or refused startup. - - What to check (list the offending changesets): - mysql -u {{ mysql_db_user }} -p -h {{ mysql_db_host }} {{ mysql_db_name }} \ - -e "SELECT ID, AUTHOR, FILENAME, DATEEXECUTED FROM DATABASECHANGELOG \ - WHERE DATEEXECUTED > FROM_UNIXTIME({{ _restore_stat.stat.mtime | int }}) \ - ORDER BY DATEEXECUTED ASC;" - - How to fix (pick one): - 1. Use a MORE RECENT backup that postdates the new migrations: - ansible-playbook ... rollback.yml -e backup_timestamp= - - 2. Restore the database from an RDS snapshot taken before - {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }}, then re-run rollback. - - 3. Override if you have already rolled back the schema by hand, - or know the new changesets are compatible with the older WAR: - ansible-playbook ... rollback.yml -e allow_downgrade=true - when: - - not (allow_downgrade | default(false) | bool) - - _changelog_count is defined - - _changelog_count.stdout is defined - - (_changelog_count.stdout | trim | int) > 0 - - - name: Warn that schema-downgrade detection was bypassed - ansible.builtin.debug: - msg: >- - ⚠ allow_downgrade=true — skipping schema-downgrade detection. - Proceeding with the rollback. Make sure the database schema is - compatible with the target WAR before continuing. - when: - - allow_downgrade | default(false) | bool - - _changelog_count is defined - - (_changelog_count.stdout | default('0') | trim | int) > 0 - - # ── 3. Stop the service ────────────────────────────────────────────────── - - - name: Stop openspecimen service - ansible.builtin.systemd: - name: openspecimen - state: stopped - - # ── 4. Restore WAR (replace the live one) ─────────────────────────────── - - - name: Remove current WAR + expanded webapp directory - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: - - "{{ tomcat_home }}/webapps/openspecimen.war" - - "{{ tomcat_home }}/webapps/openspecimen" - - - name: Restore openspecimen.war from backup - ansible.builtin.copy: - src: "{{ _restore_path }}/openspecimen.war" - dest: "{{ tomcat_home }}/webapps/openspecimen.war" - remote_src: true - owner: "{{ tomcat_user }}" - group: "{{ tomcat_user }}" - mode: '0644' - - # ── 5. Restore plugins — all three tiers ──────────────────────────────── - - - name: Probe backup plugin subdirectories - ansible.builtin.stat: - path: "{{ _restore_path }}/plugins/{{ item }}" - loop: [default, paid, zustomer] - register: _backup_plugin_stat - - - name: Wipe current plugin tier (only when backup has it) - ansible.builtin.shell: | - rm -f {{ openspecimen_plugin_dir }}/{{ item.item }}/*.jar - loop: "{{ _backup_plugin_stat.results }}" - when: item.stat.exists - changed_when: true - - - name: Restore plugin JARs from backup - ansible.builtin.shell: | - if compgen -G "{{ _restore_path }}/plugins/{{ item.item }}/*.jar" > /dev/null; then - cp {{ _restore_path }}/plugins/{{ item.item }}/*.jar {{ openspecimen_plugin_dir }}/{{ item.item }}/ - chown {{ tomcat_user }}:{{ tomcat_user }} {{ openspecimen_plugin_dir }}/{{ item.item }}/*.jar - chmod 0644 {{ openspecimen_plugin_dir }}/{{ item.item }}/*.jar - echo "restored" - else - echo "empty" - fi - args: - executable: /bin/bash - loop: "{{ _backup_plugin_stat.results }}" - when: item.stat.exists - register: _restore_plugins - changed_when: _restore_plugins.stdout == 'restored' - - # ── 6. Restore MySQL connector JAR if backed up ───────────────────────── - - - name: Probe backup lib directory - ansible.builtin.stat: - path: "{{ _restore_path }}/lib" - register: _backup_lib_stat - - - name: Restore MySQL connector JAR from backup lib/ - ansible.builtin.shell: | - if compgen -G "{{ _restore_path }}/lib/mysql-connector*.jar" > /dev/null; then - rm -f {{ tomcat_home }}/lib/mysql-connector*.jar - cp {{ _restore_path }}/lib/mysql-connector*.jar {{ tomcat_home }}/lib/ - chown {{ tomcat_user }}:{{ tomcat_user }} {{ tomcat_home }}/lib/mysql-connector*.jar - chmod 0644 {{ tomcat_home }}/lib/mysql-connector*.jar - echo "restored" - else - echo "skipped" - fi - args: - executable: /bin/bash - when: _backup_lib_stat.stat.exists - register: _restore_connector - changed_when: _restore_connector.stdout == 'restored' - - # ── 7. Restore config files from /config/ ─────────────────────── - # Captures openspecimen.properties, setenv.sh, and context.xml as they were - # at the time of the backup — no separate site.yml run needed for config. - - - name: Probe backup config files - ansible.builtin.stat: - path: "{{ _restore_path }}/config/{{ item.name }}" - loop: - - { name: openspecimen.properties, dest: "{{ tomcat_home }}/conf/openspecimen.properties", mode: '0640' } - - { name: setenv.sh, dest: "{{ tomcat_home }}/bin/setenv.sh", mode: '0755' } - - { name: context.xml, dest: "{{ tomcat_home }}/conf/context.xml", mode: '0640' } - loop_control: - label: "{{ item.name }}" - register: _backup_config_stat - changed_when: false - - - name: Restore config files from backup - ansible.builtin.copy: - src: "{{ _restore_path }}/config/{{ item.item.name }}" - dest: "{{ item.item.dest }}" - remote_src: true - owner: "{{ tomcat_user }}" - group: "{{ tomcat_user }}" - mode: "{{ item.item.mode }}" - loop: "{{ _backup_config_stat.results }}" - loop_control: - label: "{{ item.item.name }}" - when: item.stat.exists - - - name: Warn when backup has no config snapshot - ansible.builtin.debug: - msg: >- - NOTE: backup {{ _restore_path | basename }} has no config/ subdirectory. - The live openspecimen.properties / setenv.sh / context.xml are left in - place. If config was changed between this backup and now, re-run - site.yml to re-template them from inventory. - when: not (_backup_config_stat.results | map(attribute='stat.exists') | select | list | length > 0) - - # ── 8. Update the release marker to reflect the rolled-back version ───── - - - name: Probe backed-up release marker (if present) - ansible.builtin.stat: - path: "{{ _restore_path }}/.release" - register: _backup_release_marker - - - name: Restore release marker from backup - ansible.builtin.copy: - src: "{{ _restore_path }}/.release" - dest: "{{ openspecimen_release_marker }}" - remote_src: true - owner: "{{ tomcat_user }}" - group: "{{ tomcat_user }}" - mode: '0644' - when: _backup_release_marker.stat.exists - - - name: Warn when no release marker is in the backup - ansible.builtin.debug: - msg: >- - NOTE: backup {{ _restore_path | basename }} has no .release marker. - The on-disk marker still shows the previous (rolled-forward) version - and may incorrectly block the next deploy as a downgrade. If that - happens, remove {{ openspecimen_release_marker }} manually before - re-running deploy.yml. - when: not _backup_release_marker.stat.exists - - # ── 8. Start service and wait for readiness ───────────────────────────── - - - name: Start openspecimen service - ansible.builtin.systemd: - name: openspecimen - state: started - daemon_reload: true - - - name: Wait for the app to respond - ansible.builtin.shell: > - curl -sf -o /dev/null -w "%{http_code}" - http://localhost:{{ openspecimen_port }}/openspecimen/rest/ng/config-settings/app-props - | grep -qE '^(200|401)$' - register: _rollback_health - until: _rollback_health.rc == 0 - retries: 30 - delay: 10 - changed_when: false - - - name: Rollback complete - ansible.builtin.debug: - msg: >- - ✓ Rolled back to {{ _restore_path | basename }} - (mtime {{ '%Y-%m-%d %H:%M:%S' | strftime(_restore_stat.stat.mtime) }}) - on {{ inventory_hostname }}. + - name: Include rollback tasks from openspecimen role + ansible.builtin.include_role: + name: openspecimen + tasks_from: rollback diff --git a/site.yml b/site.yml index ebbab09785..3507443ed3 100644 --- a/site.yml +++ b/site.yml @@ -12,8 +12,17 @@ become: true pre_tasks: + # Detect deploy vs rollback direction by comparing openspecimen_release + # against the on-target .release marker. On downgrade, the rollback flow + # is invoked automatically and the play ends — the roles below do not run. + - name: Detect direction (deploy or rollback) + ansible.builtin.include_role: + name: openspecimen + tasks_from: direction + # Verify the release zip and every paid/customer plugin zip exists on the # control node before any role runs. Fails fast if anything is missing. + # (Only reached on forward / fresh deploys — rollback ended the play.) - name: Pre-flight artifact check ansible.builtin.include_role: name: openspecimen