diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 000000000..1d224d722 --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,444 @@ +# `kubectl dba dc-dr`: cross data center DR commands + +Day-2 operations for a KubeDB database distributed across data centers: trigger and +monitor planned switchovers, release a failover held by the RPO budget, move the +failover authority, pin a data center, and diagnose a failover that is not happening. + +Every command below was live-verified against a three data center cluster (two Member +DCs plus an Arbiter) before being documented here. + +## Which cluster each command talks to + +DC-DR spans three different API surfaces, and every command documents which one it +needs. Getting this wrong is the most common mistake, so it is stated per command in +`--help` as well. + +| surface | what lives there | how the CLI reaches it | +|---|---|---| +| **hub** | the KubeDB operator, the Postgres CRs, the PlacementPolicies | the ordinary kubeconfig flags: `--kubeconfig`, `$KUBECONFIG`, `--context`, `-n` | +| **coordination control plane** | the `primary-dc*` Leases (the failover authority) | the `--coord-*` flags below | +| **spoke** (one data center) | that DC's agent, its marker ConfigMap, and the human-owned pin ConfigMaps | the ordinary kubeconfig flags, pointed AT that spoke | + +### Reaching the coordination control plane + +It is a separate apiserver, so it needs its own kubeconfig. Commands that read or +write Leases (`active-dc`, `handoff`, `debug failover`, `debug fence`) accept, in +priority order: + +``` +--coord-kubeconfig /path/to/coord.yaml # a file +--coord-kubeconfig-configmap [namespace/]name # key "kubeconfig" in a ConfigMap on the CURRENT cluster +--coord-kubeconfig-secret [namespace/]name # key "kubeconfig" in a Secret on the CURRENT cluster + # default: dc-failover/coord-kubeconfig +--coord-namespace dc-failover # namespace holding the Leases +``` + +The default works out of the box on a standard install: the dr-controlplane chart +mints `dc-failover/coord-kubeconfig` on the hub, so pointing `$KUBECONFIG` at the hub +is usually all that is needed. + +## Command summary + +| command | what it does | talks to | +|---|---|---| +| [`switchover`](#switchover) | planned, zero-RPO move of the primary role | hub | +| [`status`](#status) | one-shot state and switchover progress | hub | +| [`abort`](#abort) | abort an in-flight switchover | hub | +| [`accept-data-loss`](#accept-data-loss) | release a failover held by the RPO budget | hub | +| [`handoff`](#handoff) | move the failover authority (the failover lever) | hub + coordination | +| [`pin-primary`](#pin-primary) | break-glass: this DC stays primary, no failover | spoke | +| [`pin-standby`](#pin-standby) | this DC never promotes | spoke | +| [`active-dc`](#active-dc) | who holds the primary role right now | hub + coordination | +| [`debug`](#debug) | diagnose failover / switchover / fence symptoms | hub + coordination | + +--- + +## switchover + +Planned, zero-RPO move of the primary role to another data center. + +```sh +kubectl dba dc-dr switchover DB_NAME -n NS --to DC +``` + +Sets `dr.kubedb.com/switchover-to`. The operator quiesces the active primary +(write-locks it so its LSN freezes), waits for the target to replay to that exact +LSN, hands off the Lease, and clears the annotation. No committed row is lost. + +**Requires the active primary to be up and reachable.** All three of the operator's +gates measure by dialing it and fail closed, so a dead primary can never be switched +away from; use [`handoff`](#handoff) for that case. + +Refuses up front: a target that is not a Member DC (an Arbiter or Witness can never +hold the primary role), and a target that is already active (prints a no-op notice). + +``` +$ kubectl dba dc-dr switchover cli-test -n demo --to dc-a +Switchover of demo/cli-test to "dc-a" requested (scope primary-dc-clitest, from PlacementPolicy cli-test-pp failoverPolicy trigger (Group "clitest")). +The operator will quiesce, wait for catch-up, and hand off; zero committed rows are lost. +Monitor: kubectl dba dc-dr status cli-test -n demo +Abort: kubectl dba dc-dr abort cli-test -n demo + +$ kubectl dba dc-dr switchover cli-test -n demo --to dc-c +Error: "dc-c" is not a Member data center of this database (members: [dc-a dc-b]); an Arbiter or Witness DC can never become primary +``` + +If the database's scope is Global, the command says so: every database in that scope +switches with it. + +## status + +One-shot DC-DR picture, plus step-by-step switchover progress when one is in flight. + +```sh +kubectl dba dc-dr status DB_NAME -n NS +``` + +**It does not follow.** Run it again to see the next state; that keeps the output +pasteable into a ticket. The step states are derived from the same status fields the +operator's own gates read, so the display cannot drift from the real decision. + +``` +$ kubectl dba dc-dr status pg-dcdr -n demo +Database: demo/pg-dcdr (Ready) +Failover scope: primary-dc + (PlacementPolicy dcdr-postgres failoverPolicy trigger (Global)) +Member DCs: dc-a, dc-b +Active DC: dc-b DR phase: Steady +Protected: true (data center "dc-a" is streaming 0 bytes behind, within the 16777216 byte budget) + +Data centers: + NAME ROLE WRITABLE HEALTHY LAG(BYTES) STREAMER + dc-a Member - true 0 pg-dcdr-dc-a-1 + dc-b Member true true - pg-dcdr-dc-b-1 + dc-c Arbiter - true - + +No switchover in flight. +Trigger one: kubectl dba dc-dr status pg-dcdr -n demo --to +``` + +During a switchover, the progress block replaces that last section: + +``` +Planned switchover to "dc-a", started 11s ago: + [done] 1. target "dc-a" validated: healthy and lag known, within the switchover budget + [done] 2. quiesce requested on the active DC (dc-b) + [NOW] 3. quiesce IN EFFECT: active primary write-locked, its LSN frozen + [pending] 4. target caught up to the frozen LSN (now 0 bytes, needs <= 8192) + [pending] 5. primary-DC Lease handed off to "dc-a" + [pending] 6. old DC demoted to standby, annotations cleared, DR phase back to Steady + + NEXT: waiting for the write-lock to take hold on dc-b. This needs the active primary to be UP + and reachable; a dead primary can never satisfy it (use the failover path). + + Abort: kubectl dba dc-dr abort cli-test -n demo + Re-run this command to see the next step; it does not follow. +``` + +## abort + +```sh +kubectl dba dc-dr abort DB_NAME -n NS +``` + +Sets `dr.kubedb.com/switchover-abort`; its PRESENCE is the signal. The hub clears the +quiesce so the original active DC resumes writes, then removes every switchover +annotation. + +**Do not abort by deleting `switchover-to`.** In a scope shared by several databases +the hub re-propagates that annotation to every sibling on each pass, so a bare removal +silently reappears. A switchover that cannot progress also auto-aborts on its own after +`dr.kubedb.com/switchover-timeout` (default 10m). + +## accept-data-loss + +Releases a cross-DC failover that the RPO budget is holding. + +```sh +kubectl dba dc-dr accept-data-loss DB_NAME -n NS --yes +``` + +When the surviving DC lags more than `spec.replication.bestEffortCrossDCLagBytesForFailover` +(or its lag cannot be measured), promotion is **held**: the lost DC's un-replicated WAL +is unrecoverable, so the choice between an outage and a larger-than-budgeted loss +belongs to a human. This records that decision. + +The command prints the current protection verdict first, and `--yes` is mandatory. Both +promotion paths (the hub gate and the coordinator's data-plane gate) honor it within +seconds, and the operator removes the annotation automatically once the failover it +authorized lands, so it cannot linger and approve a later, unrelated loss. + +``` +$ kubectl dba dc-dr accept-data-loss cli-test -n demo --yes +Current protection verdict: data center "dc-b" is streaming 0 bytes behind, within the 16777216 byte budget +NOTE: the database currently reads protected=true; nothing seems held. Setting the annotation is still +safe: it is only honored by a promotion that the budget is actively refusing, and it is auto-removed after use. +Data-loss acceptance recorded on demo/cli-test. +``` + +## handoff + +Moves the failover authority by annotating the Lease itself. + +```sh +kubectl dba dc-dr handoff (DB_NAME | --lease NAME) --to DC --yes +``` + +This is the **scope-local failover lever**, and the right tool when the active DC's +database is down but its data center is alive (nothing fails over on its own in that +state: the agent renews the Lease on its own health, not the database's). No quiesce +and no catch-up wait happen, so loss is bounded by the RPO budget rather than zero. +For a healthy primary prefer [`switchover`](#switchover). + +It moves **every** database in the scope. Never stop a DC's agent to force a failover +instead: one agent serves every scope that DC holds, so stopping it expires all of them. + +`--lease` works with no database present, which is how orphan scopes are moved. +Refuses a non-Member target, and refuses outright while a break-glass pin holds the +scope. Re-issuing the same target is safe: the command clears and re-sets the +annotation so the agents always see a real transition. + +``` +$ kubectl dba dc-dr handoff cli-test -n demo --to dc-b --yes +Handoff of primary-dc-clitest requested: dc-a -> dc-b. +The holder releases within seconds and dc-b acquires on its next retry tick; the annotation clears itself. +Verify: kubectl dba dc-dr active-dc --lease primary-dc-clitest +``` + +Measured: the target held the role about 6 seconds later. + +## pin-primary + +Break-glass. This data center stays primary and stays writable, even through a +coordination-plane outage. + +```sh +# run against the SPOKE of the DC being pinned +kubectl dba dc-dr pin-primary (--scope LEASE | --db DB) --yes +kubectl dba dc-dr pin-primary --scope LEASE --remove --yes +# the pinned DC is DEAD and its pin is stuck: run from any live cluster +kubectl dba dc-dr pin-primary --scope LEASE --remove --force --yes --coord-kubeconfig coord.yaml +``` + +Creates the human-owned `-override` ConfigMap on that DC's spoke, which is +precisely why it still works when the hub is unreachable. Two effects: the DC's agent +mirrors the pin onto the Lease so every other Member defers permanently, and the DC's +coordinator forces its leader active regardless of marker state, so a sustained +control-plane outage no longer fences it read-only. + +Without `--yes` it prints what it would do and exits non-zero. + +``` +$ kubectl dba dc-dr pin-primary --scope primary-dc-clitest --yes # against dc-b +ConfigMap dc-failover/primary-dc-clitest-override created on the current cluster. +This data center is now PINNED primary for scope primary-dc-clitest: + - its agent mirrors the pin onto the Lease, so no other Member contends; + - its coordinator keeps the local leader writable even if the marker goes stale. + IMPORTANT: only honored on the scope's LAST KNOWN HOLDER. On any other DC the agent refuses it. + IMPORTANT: while pinned, that DC dying means NO failover happens. +``` + +The pin is then visible from anywhere, and blocks a handoff: + +``` +$ kubectl dba dc-dr active-dc --lease primary-dc-clitest +Active DC: dc-b +PINNED: break-glass override holds this scope on dc-b; it cannot fail over until the + override ConfigMap is removed from that DC's spoke. + +$ kubectl dba dc-dr handoff --lease primary-dc-clitest --to dc-a --yes +Error: scope primary-dc-clitest is PINNED to "dc-b" by a break-glass override; remove that DC's +override ConfigMap first (kubectl dba dc-dr pin-primary --remove), or the handoff cannot complete +``` + +Caveats, all live-verified: it is honored **only on the scope's last known holder** (on +any other DC the agent refuses it and logs why, so it never promotes a standby); while +it stands there is no split-brain protection for the scope and nothing takes over if +that DC dies; and on control-plane recovery the recreated Lease is claimed first-come, +so keep the pin until the authority is back with the right holder, then remove it. + +### --remove --force: the pinned DC died with its pin standing + +The override-hold annotation on the Lease is normally cleared only by the pinned DC's +own agent (when it sees its ConfigMap gone). If that DC is dead, the annotation stays +and every handoff refuses. `--remove --force` breaks that deadlock: run it from **any +live cluster** with the `--coord-*` flags; the ConfigMap delete on the current cluster +is attempted, a failure is reported and skipped, and the `override-hold` annotation is +then cleared directly on the Lease. Live-proven in the bank dc-loss drill: + +``` +$ kubectl dba dc-dr pin-primary --scope primary-dc-bankpg --remove --force --yes \ + --coord-kubeconfig /tmp/bank-coord.yaml # from dr, dc dead +Could not remove ConfigMap dc-failover/primary-dc-bankpg-override on the current cluster +(configmaps "primary-dc-bankpg-override" not found); proceeding to clear the Lease +annotation because --force is set. +Cleared the override-hold annotation on Lease dc-failover/primary-dc-bankpg directly +(the pinned DC's agent is not alive to do it). +Members now contend normally; the surviving DC can acquire once its holds are removed. +``` + +ONLY for a dead DC. If the pinned DC is alive, its coordinator still honors the local +ConfigMap (it keeps its leader writable), so force-clearing just the annotation while +another DC promotes is a split brain. Alive DC: plain `--remove` on its spoke first. + +The stale ConfigMap on the dead cluster is a live grenade for failback: the returning +agent refuses to mirror it (no longer the last known holder), but the returning +COORDINATOR honors it unconditionally and brings the old primary back up writable +while the other DC holds the Lease. Live-observed as a real two-primaries window in +the bank drill. Failback step 1, before restoring the database workload on the +returning DC: `pin-primary --scope LEASE --remove --yes` against that spoke. + +## pin-standby + +This data center never promotes. + +```sh +# run against the SPOKE of the DC being held +kubectl dba dc-dr pin-standby (--scope LEASE | --db DB) --yes +kubectl dba dc-dr pin-standby --scope LEASE --remove --yes +``` + +Creates `-standby-hold` on that DC's spoke. While it exists the DC never +contends for the Lease, never promotes (it refuses **even an explicit handoff naming +it**, verified live), and its coordinator refuses destructive cross-DC rewinds of the +data it holds. It fails **closed**: an unreadable ConfigMap is treated as held, so a +flaky apiserver never silently drops the protection. + +It is deliberately ignored on the DC that is currently active, because demoting the +active DC without a quiesce is unsafe. Move the primary away with a switchover first. + +Removing the hold takes effect immediately: the agent re-evaluates that scope's +contention the moment the ConfigMap disappears. Verified live, a handoff that had been +vetoed for 30s completed 10 seconds after the hold was released, with no further +action. + +Timing caveat, live-burned in the bank failback drill: pin only a DC that is already +streaming. A DC that just lost a failover still needs its one-time rewind onto the +winner's timeline, and the hold refuses exactly that (by design, "data untouched"), +so pinning too early leaves the DC down and fenced instead of standby. Re-apply the +hold only once `pg_stat_replication` on the primary shows that DC's leader. + +## active-dc + +Who holds the primary role, from the authority itself. + +```sh +kubectl dba dc-dr active-dc DB_NAME -n NS +kubectl dba dc-dr active-dc --lease primary-dc-orders +kubectl dba dc-dr active-dc DB_NAME -n NS -q # just the DC name, for scripts +``` + +Given a database, its scope is resolved exactly as the operator resolves it (the +PlacementPolicy's `failoverPolicy.trigger`, falling back to the legacy +`dr.kubedb.com/failover-group` annotation, else Global) and the matching Lease is read. + +``` +$ kubectl dba dc-dr active-dc pg-dcdr -n demo +Active DC: dc-b +Lease: dc-failover/primary-dc (scope from PlacementPolicy dcdr-postgres failoverPolicy trigger (Global)) +Renewed: 2s ago (lease duration 45s) +Transitions:41 +Members: dc-a,dc-b +``` + +It warns when the Lease is expired, shows an in-flight handoff, shows a break-glass +pin, and notes when the database's own `status.activeDC` disagrees (the status trails +the Lease by a reconcile; the Lease is the authority). + +## debug + +```sh +kubectl dba dc-dr debug failover DB_NAME -n NS # nothing failed over +kubectl dba dc-dr debug switchover DB_NAME -n NS # switchover not completing +kubectl dba dc-dr debug fence DB_NAME -n NS # database up but read-only +``` + +Each walks the real decision chain and reports every condition in order, marking +`OK` / `WARN` / `FAIL`, with a concrete remedy on anything that is not OK, and a count +of blocking conditions at the end. + +`debug failover` checks: the database is distributed and armed; the scope is +registered (an unregistered scope has no Lease, so nothing can ever move); the Lease +exists, who holds it, and whether it is still being renewed; break-glass pins and +in-flight handoffs; each candidate DC's health Lease; the RPO budget verdict; stale or +failed ForceFailOver ops; and the DC-DR conditions including the retry cap. + +Real output from a database whose failovers had been failing overnight: + +``` +$ kubectl dba dc-dr debug failover pg-dcdr -n demo +Diagnosing failover for demo/pg-dcdr + + [OK ] database is DC-DR distributed and armed + [OK ] failover scope resolves to primary-dc + PlacementPolicy dcdr-postgres failoverPolicy trigger (Global) + [WARN] holder "dc-b" is renewing normally, so the authority will NOT move on its own + the Lease was renewed 5s ago, inside its 45s duration: that data center's agent is alive + -> this is by design: no database-level condition (client errors, QPS, lag, a crashed + postgres) ever moves the Lease. If the DATABASE is down but the DC is alive, either let + the DC's own raft promote a local peer, or move the scope deliberately: + kubectl dba dc-dr handoff pg-dcdr -n demo --to --yes + [OK ] no break-glass pin on the Lease + [OK ] candidate DC "dc-a" is alive (health renewed 0s ago) + [OK ] protection is confirmed (RPO budget satisfied) + [WARN] no failed ForceFailOver ops are blocking + failed: pg-dcdr-dcdr-failover-92vjc, ... (5 total) + -> read their status for the real cause, then delete them + [FAIL] the ForceFailOver retry cap is not tripped + 5 consecutive ForceFailOver attempts to promote data center "dc-b" failed; the hub has + stopped creating new attempts and needs manual intervention + -> Fix the underlying failure, delete the failed ops, and it resumes + +1 blocking condition(s) found. +``` + +`debug switchover` reports which gate is holding, and calls out the case people hit +most: every gate measures by dialing the active primary, so an unreachable primary +stalls the switchover permanently (abort and use `handoff`). + +`debug fence` explains a database that is up but refusing writes: whether the +authority is healthy, whether the Lease is being renewed (a stale Lease means every +DC's marker goes stale and all of them fence, by design), and what to do when the +coordination plane itself is the broken thing. + +--- + +## Worked examples + +**Planned move, monitored.** + +```sh +kubectl dba dc-dr switchover pg-dcdr -n demo --to dc-a +kubectl dba dc-dr status pg-dcdr -n demo # re-run until step 6 is done +``` + +**The active DC's database is dead but the DC is alive.** Nothing fails over on its +own; that is by design. Confirm, then move the scope: + +```sh +kubectl dba dc-dr debug failover pg-dcdr -n demo +kubectl dba dc-dr handoff pg-dcdr -n demo --to dc-b --yes +kubectl dba dc-dr active-dc pg-dcdr -n demo +``` + +**A failover is held by the RPO budget.** + +```sh +kubectl dba dc-dr status pg-dcdr -n demo # read protectionMessage +kubectl dba dc-dr accept-data-loss pg-dcdr -n demo --yes +``` + +**The coordination plane is down and the primary must keep writing.** Run against the +active DC's spoke, before its marker goes stale (roughly 90s from the last renewal): + +```sh +kubectl dba dc-dr pin-primary --scope primary-dc --yes --kubeconfig ~/.kube/dc-b.yaml +# ... after recovery, once the Lease is back with the right holder: +kubectl dba dc-dr pin-primary --scope primary-dc --remove --yes --kubeconfig ~/.kube/dc-b.yaml +``` + +**Keep a data center out of the primary role permanently.** + +```sh +kubectl dba dc-dr pin-standby --scope primary-dc --yes --kubeconfig ~/.kube/dc-a.yaml +``` diff --git a/pkg/cmds/dcdr.go b/pkg/cmds/dcdr.go new file mode 100644 index 000000000..60bcedef8 --- /dev/null +++ b/pkg/cmds/dcdr.go @@ -0,0 +1,26 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmds + +import ( + "kubedb.dev/cli/pkg/dcdr" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +// NewCmdDCDR returns the cross data center DR command group. +func NewCmdDCDR(f cmdutil.Factory) *cobra.Command { + return dcdr.NewCmdDCDR(f) +} diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index a222925bb..1ab74a0f1 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -114,6 +114,12 @@ func NewKubeDBCommand(in io.Reader, out, err io.Writer) *cobra.Command { NewCmdMSSQL(f), }, }, + { + Message: "Cross Data Center DR Commands", + Commands: []*cobra.Command{ + NewCmdDCDR(f), + }, + }, { Message: "Metric related CMDs", Commands: []*cobra.Command{ diff --git a/pkg/dcdr/abort.go b/pkg/dcdr/abort.go new file mode 100644 index 000000000..09fb82af4 --- /dev/null +++ b/pkg/dcdr/abort.go @@ -0,0 +1,69 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdAbort aborts an in-flight planned switchover. Acts on the HUB cluster. +func NewCmdAbort(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "abort DB_NAME", + Short: i18n.T("Abort an in-flight planned switchover and restore writes to the current active DC"), + Long: templates.LongDesc(` + Sets the dr.kubedb.com/switchover-abort annotation. Its PRESENCE aborts: + the hub clears the quiesce so the original active data center resumes + accepting writes, and removes every switchover annotation once done. + + Do NOT abort by deleting the switchover-to annotation: in a scope shared + by several databases the hub re-propagates it to every sibling each pass, + so a bare removal silently reappears. This explicit abort signal is + propagated and honored scope-wide. A switchover that cannot complete also + auto-aborts on its own after the switchover timeout (default 10m, + dr.kubedb.com/switchover-timeout to override). + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(` + kubectl dba dc-dr abort pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + if db.GetAnnotations()[AnnSwitchoverTo] == "" && db.GetAnnotations()[AnnQuiesceActive] == "" { + cmd.Printf("No switchover appears to be in flight on %s/%s (no %s or %s annotation); setting the abort anyway is harmless.\n", ns, args[0], AnnSwitchoverTo, AnnQuiesceActive) + } + v := "true" + if err := annotateDB(ctx, dyn, ns, args[0], AnnSwitchoverAbort, &v); err != nil { + return err + } + cmd.Printf("Abort requested for %s/%s. The hub restores writes to the current active DC and clears the switchover annotations; verify with:\n", ns, args[0]) + cmd.Printf(" kubectl dba dc-dr status %s -n %s\n", args[0], ns) + return nil + }, + } + return cmd +} diff --git a/pkg/dcdr/acceptloss.go b/pkg/dcdr/acceptloss.go new file mode 100644 index 000000000..9bfff8a11 --- /dev/null +++ b/pkg/dcdr/acceptloss.go @@ -0,0 +1,89 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdAcceptDataLoss releases a cross-DC failover held by the RPO budget. Acts +// on the HUB cluster. +func NewCmdAcceptDataLoss(f cmdutil.Factory) *cobra.Command { + var yes bool + cmd := &cobra.Command{ + Use: "accept-data-loss DB_NAME --yes", + Short: i18n.T("Release a failover held by the RPO budget, explicitly accepting the data loss"), + Long: templates.LongDesc(` + When the surviving data center lags more than + spec.replication.bestEffortCrossDCLagBytesForFailover (or its lag cannot + be measured), the promotion is HELD: the un-replicated WAL of the lost + data center is unrecoverable, so choosing between an outage and a loss + larger than the budget belongs to a human. This command records that + decision by setting dr.kubedb.com/accept-failover-data-loss=true; both + promotion paths (the hub gate and the coordinator's data-plane gate) + honor it within seconds, and the operator removes the annotation + automatically once the failover it authorized lands, so it cannot linger + and approve a later, unrelated loss. + + Where the hold is visible before you decide: + status.disasterRecovery.protectionMessage (the measured lag), + condition DCDRPromotionStalled, and dc-dr status. + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(` + # See what would be lost first + kubectl dba dc-dr status pg-dcdr -n demo + + # Accept it + kubectl dba dc-dr accept-data-loss pg-dcdr -n demo --yes`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + msg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage") + protected, protectedFound, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected") + if msg != "" { + cmd.Printf("Current protection verdict: %s\n", msg) + } + if protectedFound && protected { + cmd.Printf("NOTE: the database currently reads protected=true; nothing seems held. Setting the annotation is still safe: it is only honored by a promotion that the budget is actively refusing, and it is auto-removed after use.\n") + } + if !yes { + return fmt.Errorf("this authorizes losing committed data beyond the configured budget; re-run with --yes to confirm") + } + v := "true" + if err := annotateDB(ctx, dyn, ns, args[0], AnnAcceptDataLoss, &v); err != nil { + return err + } + cmd.Printf("Data-loss acceptance recorded on %s/%s. The held promotion proceeds within seconds; the annotation is removed automatically once the failover lands.\n", ns, args[0]) + cmd.Printf("Monitor: kubectl dba dc-dr status %s -n %s\n", args[0], ns) + return nil + }, + } + cmd.Flags().BoolVar(&yes, "yes", false, "Confirm accepting data loss beyond the configured RPO budget") + return cmd +} diff --git a/pkg/dcdr/activedc.go b/pkg/dcdr/activedc.go new file mode 100644 index 000000000..988ea15ba --- /dev/null +++ b/pkg/dcdr/activedc.go @@ -0,0 +1,138 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdActiveDC answers "who is primary" from the authoritative source, the Lease +// on the coordination plane, resolving the scope from a database name when given. +func NewCmdActiveDC(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + var leaseName string + var quiet bool + cmd := &cobra.Command{ + Use: "active-dc [DB_NAME] [--lease NAME]", + Short: i18n.T("Print the data center that currently holds the primary role"), + Long: templates.LongDesc(` + Reads the primary-DC Lease from the coordination control plane, the + authority for which data center is active. + + Given a database name, its failover scope is resolved first (the + PlacementPolicy's failoverPolicy trigger, exactly as the operator + resolves it) and the matching Lease is read. Given --lease, that Lease is + read directly, which also works for a scope whose database is gone. + + KUBECONFIG: the hub cluster (to read the Postgres and its + PlacementPolicy, and by default to read the coordination kubeconfig + Secret). The coordination plane itself is reached with the --coord-* + flags.`), + Example: templates.Examples(` + # By database + kubectl dba dc-dr active-dc pg-dcdr -n demo + + # By Lease name, with an explicit coordination kubeconfig file + kubectl dba dc-dr active-dc --lease primary-dc --coord-kubeconfig /tmp/coord.yaml + + # Scriptable: just the DC name + kubectl dba dc-dr active-dc pg-dcdr -n demo -q`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if (len(args) == 0) == (leaseName == "") { + return fmt.Errorf("give exactly one of: a database name, or --lease") + } + ctx := context.Background() + out := cmd.OutOrStdout() + scope := &Scope{LeaseName: leaseName, Source: "--lease flag"} + var ns string + if len(args) == 1 { + db, dyn, dbNS, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + ns = dbNS + scope, err = ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + } + coord, err := cf.CoordClient(ctx, f) + if err != nil { + return err + } + lease, err := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read Lease %s/%s on the coordination plane: %w", cf.LeaseNS, scope.LeaseName, err) + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + if quiet { + _, _ = fmt.Fprintln(out, holder) + return nil + } + _, _ = fmt.Fprintf(out, "Active DC: %s\n", orNone(holder)) + _, _ = fmt.Fprintf(out, "Lease: %s/%s (scope from %s)\n", cf.LeaseNS, scope.LeaseName, scope.Source) + if lease.Spec.RenewTime != nil { + age := time.Since(lease.Spec.RenewTime.Time).Round(time.Second) + dur := int32(0) + if lease.Spec.LeaseDurationSeconds != nil { + dur = *lease.Spec.LeaseDurationSeconds + } + _, _ = fmt.Fprintf(out, "Renewed: %s ago (lease duration %ds)\n", age, dur) + if dur > 0 && age > time.Duration(dur)*time.Second { + _, _ = fmt.Fprintf(out, " WARNING: the Lease is EXPIRED. Its holder stopped renewing, so a healthy Member DC may acquire it at any moment.\n") + } + } + if lease.Spec.LeaseTransitions != nil { + _, _ = fmt.Fprintf(out, "Transitions:%d\n", *lease.Spec.LeaseTransitions) + } + if v := lease.Annotations[AnnLeaseMemberDCs]; v != "" { + _, _ = fmt.Fprintf(out, "Members: %s\n", v) + } + if v := lease.Annotations[AnnLeaseHandoffTo]; v != "" { + _, _ = fmt.Fprintf(out, "Handoff to: %s (a coordinated handoff is in flight)\n", v) + } + if v := lease.Annotations["dr.open-cluster-management.io/override-hold"]; v != "" { + _, _ = fmt.Fprintf(out, "PINNED: break-glass override holds this scope on %s; it cannot fail over until the override ConfigMap is removed from that DC's spoke.\n", v) + } + if len(args) == 1 { + // Cross-check the CR's own view, which lags the Lease by a reconcile. + db, _, _, err := getDB(ctx, f, args[0]) + if err == nil { + crActive, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + if crActive != "" && crActive != holder { + _, _ = fmt.Fprintf(out, "\nNOTE: the database %s/%s still reports activeDC=%s in its status; the Lease is the authority and the status trails it by a reconcile.\n", ns, args[0], crActive) + } + } + } + return nil + }, + } + cmd.Flags().StringVar(&leaseName, "lease", "", "Read this Lease directly instead of resolving a database's scope") + cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Print only the active DC name") + AddCoordFlags(cmd, &cf) + return cmd +} diff --git a/pkg/dcdr/dcdr.go b/pkg/dcdr/dcdr.go new file mode 100644 index 000000000..c820eaa71 --- /dev/null +++ b/pkg/dcdr/dcdr.go @@ -0,0 +1,341 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package dcdr implements `kubectl dba dc-dr`, the day-2 command group for KubeDB +// cross data center disaster recovery (DC-DR). +// +// The commands act on up to three different control planes, and each one documents +// which: +// +// - The HUB cluster (where the KubeDB operator and the database CRs live): the +// ordinary kubeconfig flags / $KUBECONFIG select it, exactly like every other +// kubectl-dba command. +// - The COORDINATION control plane (a separate apiserver holding the primary-DC +// Leases): reached with a dedicated kubeconfig resolved by CoordFlags, from a +// file, a Secret, or a ConfigMap (see AddCoordFlags). +// - A SPOKE cluster (one data center): the pin commands create their marker +// ConfigMaps on a specific DC's spoke, so for those the ordinary kubeconfig +// flags must point AT that spoke. +package dcdr + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// GroupVersionResources of the objects this command group touches. Everything is +// accessed unstructured on purpose: the DC-DR status and PlacementPolicy types are +// not part of the released apimachinery this CLI vendors, and the commands only +// need a handful of well-known paths. +var ( + PostgresGVR = schema.GroupVersionResource{Group: "kubedb.com", Version: "v1", Resource: "postgreses"} + PlacementGVR = schema.GroupVersionResource{Group: "apps.k8s.appscode.com", Version: "v1", Resource: "placementpolicies"} + PgOpsGVR = schema.GroupVersionResource{Group: "ops.kubedb.com", Version: "v1alpha1", Resource: "postgresopsrequests"} +) + +// The annotation and naming contract shared with the KubeDB Postgres operator and +// the dr-controlplane service. Keep byte for byte in sync with +// postgres/pkg/dcdr/helpers.go and dr-controlplane/pkg/leases/names.go. +const ( + AnnSwitchoverTo = "dr.kubedb.com/switchover-to" + AnnSwitchoverAbort = "dr.kubedb.com/switchover-abort" + AnnSwitchoverStart = "dr.kubedb.com/switchover-started" + AnnQuiesceActive = "dr.kubedb.com/quiesce-active" + AnnAcceptDataLoss = "dr.kubedb.com/accept-failover-data-loss" + AnnFailoverGroup = "dr.kubedb.com/failover-group" + AnnMaxLagBytes = "dr.kubedb.com/switchover-max-lag-bytes" + + AnnLeaseHandoffTo = "dr.open-cluster-management.io/handoff-to" + AnnLeaseMemberDCs = "dr.open-cluster-management.io/member-dcs" + + GlobalPrimaryLease = "primary-dc" + GroupLeasePrefix = "primary-dc-" + + OverrideCMSuffix = "-override" + StandbyHoldCMSuffix = "-standby-hold" + + // DefaultCoordNamespace is where the Leases and marker ConfigMaps live, on the + // coordination plane and on every spoke respectively. + DefaultCoordNamespace = "dc-failover" + + // zeroRPOLagBytes mirrors the operator's switchover residual-lag tolerance. + zeroRPOLagBytes = int64(8 * 1024) +) + +// NewCmdDCDR returns the `dc-dr` command group. +func NewCmdDCDR(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "dc-dr", + Short: i18n.T("Cross data center DR operations: switchover, failover, pins, and diagnosis"), + Long: templates.LongDesc(` + Operate a KubeDB database that is distributed across data centers: + trigger and monitor planned switchovers, accept a held failover's data + loss, move the failover authority, pin a data center, and diagnose a + failover that is not happening.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand( + NewCmdSwitchover(f), + NewCmdStatus(f), + NewCmdAbort(f), + NewCmdAcceptDataLoss(f), + NewCmdDebug(f), + NewCmdHandoff(f), + NewCmdPin(f, pinPrimary), + NewCmdPin(f, pinStandby), + NewCmdActiveDC(f), + ) + return cmd +} + +// CoordFlags resolves the kubeconfig of the coordination control plane, the +// separate apiserver that stores the primary-DC Leases. Resolution order: +// +// 1. --coord-kubeconfig: a kubeconfig FILE for the coordination plane. +// 2. --coord-kubeconfig-configmap [ns/]name: read key "kubeconfig" from that +// ConfigMap on the CURRENT cluster. +// 3. --coord-kubeconfig-secret [ns/]name (default dc-failover/coord-kubeconfig, +// the name the dr-controlplane chart mints): read key "kubeconfig" from that +// Secret on the CURRENT cluster. +type CoordFlags struct { + File string + Secret string + ConfigMap string + // LeaseNS is the namespace holding the Leases on the coordination plane. + LeaseNS string +} + +// AddCoordFlags registers the coordination-plane kubeconfig flags on cmd. +func AddCoordFlags(cmd *cobra.Command, cf *CoordFlags) { + cmd.Flags().StringVar(&cf.File, "coord-kubeconfig", "", "Path to a kubeconfig file for the coordination control plane (overrides the secret/configmap sources)") + cmd.Flags().StringVar(&cf.Secret, "coord-kubeconfig-secret", DefaultCoordNamespace+"/coord-kubeconfig", "Secret ([namespace/]name, key \"kubeconfig\") on the current cluster holding the coordination-plane kubeconfig") + cmd.Flags().StringVar(&cf.ConfigMap, "coord-kubeconfig-configmap", "", "ConfigMap ([namespace/]name, key \"kubeconfig\") on the current cluster holding the coordination-plane kubeconfig") + cmd.Flags().StringVar(&cf.LeaseNS, "coord-namespace", DefaultCoordNamespace, "Namespace on the coordination plane that holds the primary-DC Leases") +} + +func splitNSName(s, defaultNS string) (ns, name string) { + if ns, name, found := strings.Cut(s, "/"); found { + return ns, name + } + return defaultNS, s +} + +// CoordClient builds a client for the coordination control plane per the CoordFlags +// resolution order. The Secret/ConfigMap sources are read through f, the CURRENT +// cluster (normally the hub). +func (cf *CoordFlags) CoordClient(ctx context.Context, f cmdutil.Factory) (kubernetes.Interface, error) { + var kubeconfigBytes []byte + switch { + case cf.File != "": + b, err := os.ReadFile(cf.File) + if err != nil { + return nil, fmt.Errorf("failed to read --coord-kubeconfig file: %w", err) + } + kubeconfigBytes = b + case cf.ConfigMap != "": + cur, err := f.KubernetesClientSet() + if err != nil { + return nil, err + } + ns, name := splitNSName(cf.ConfigMap, DefaultCoordNamespace) + cm, err := cur.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to read coordination kubeconfig ConfigMap %s/%s from the current cluster: %w", ns, name, err) + } + kubeconfigBytes = []byte(cm.Data["kubeconfig"]) + default: + cur, err := f.KubernetesClientSet() + if err != nil { + return nil, err + } + ns, name := splitNSName(cf.Secret, DefaultCoordNamespace) + sec, err := cur.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to read coordination kubeconfig Secret %s/%s from the current cluster (is the current kubeconfig pointing at the hub?): %w", ns, name, err) + } + kubeconfigBytes = sec.Data["kubeconfig"] + } + if len(kubeconfigBytes) == 0 { + return nil, fmt.Errorf("resolved an empty coordination-plane kubeconfig (expected key \"kubeconfig\")") + } + cfg, err := clientcmd.RESTConfigFromKubeConfig(kubeconfigBytes) + if err != nil { + return nil, fmt.Errorf("coordination-plane kubeconfig is not valid: %w", err) + } + return kubernetes.NewForConfig(cfg) +} + +// Scope is a database's failover scope resolved to its Lease name. +type Scope struct { + LeaseName string + // Source explains where the scope came from, for human output. + Source string + // MemberDCs are the data-bearing Member DCs from the PlacementPolicy, empty + // when the policy was not resolvable. + MemberDCs []string +} + +// ResolveScopeForDB mirrors the operator's scopeForDB: the PlacementPolicy's +// failoverPolicy.trigger is the source of truth (Group with a name, else Global); +// the dr.kubedb.com/failover-group annotation is consulted only when the policy +// carries no failoverPolicy; otherwise the scope is Global. +func ResolveScopeForDB(ctx context.Context, dyn dynamic.Interface, db *unstructured.Unstructured) (*Scope, error) { + ppName, _, _ := unstructured.NestedString(db.Object, "spec", "podTemplate", "spec", "podPlacementPolicy", "name") + if ppName != "" { + pp, err := dyn.Resource(PlacementGVR).Get(ctx, ppName, metav1.GetOptions{}) + if err == nil { + s := &Scope{MemberDCs: memberDCsFromPP(pp)} + trigger, found, _ := unstructured.NestedMap(pp.Object, "spec", "clusterSpreadConstraint", "failoverPolicy", "trigger") + if found { + scope, _ := trigger["scope"].(string) + group, _ := trigger["group"].(string) + if scope == "Group" && group != "" { + s.LeaseName = GroupLeasePrefix + group + s.Source = fmt.Sprintf("PlacementPolicy %s failoverPolicy trigger (Group %q)", ppName, group) + return s, nil + } + s.LeaseName = GlobalPrimaryLease + s.Source = fmt.Sprintf("PlacementPolicy %s failoverPolicy trigger (Global)", ppName) + return s, nil + } + // Policy exists but registers no failoverPolicy: back-compat annotation. + if g := db.GetAnnotations()[AnnFailoverGroup]; g != "" { + s.LeaseName = GroupLeasePrefix + g + s.Source = fmt.Sprintf("annotation %s (PlacementPolicy %s has no failoverPolicy)", AnnFailoverGroup, ppName) + return s, nil + } + s.LeaseName = GlobalPrimaryLease + s.Source = fmt.Sprintf("default Global (PlacementPolicy %s has no failoverPolicy); WARNING: this scope may not be registered, so no Lease may exist and protection may not be armed", ppName) + return s, nil + } + // The policy is referenced but unreadable: fall through to the annotation, + // but say so. + if g := db.GetAnnotations()[AnnFailoverGroup]; g != "" { + return &Scope{LeaseName: GroupLeasePrefix + g, Source: fmt.Sprintf("annotation %s (PlacementPolicy %s unreadable: %v)", AnnFailoverGroup, ppName, err)}, nil + } + return &Scope{LeaseName: GlobalPrimaryLease, Source: fmt.Sprintf("default Global (PlacementPolicy %s unreadable: %v)", ppName, err)}, nil + } + if g := db.GetAnnotations()[AnnFailoverGroup]; g != "" { + return &Scope{LeaseName: GroupLeasePrefix + g, Source: "annotation " + AnnFailoverGroup}, nil + } + return &Scope{LeaseName: GlobalPrimaryLease, Source: "default Global (no PlacementPolicy set)"}, nil +} + +func memberDCsFromPP(pp *unstructured.Unstructured) []string { + rules, _, _ := unstructured.NestedSlice(pp.Object, "spec", "clusterSpreadConstraint", "distributionRules") + var members []string + for _, r := range rules { + rm, ok := r.(map[string]any) + if !ok { + continue + } + name, _ := rm["clusterName"].(string) + indices, _ := rm["replicaIndices"].([]any) + role, _ := rm["role"].(string) + // A data-bearing Member: explicit role Member, or (older policies) any rule + // with replicaIndices. Arbiter/Witness DCs never hold the primary. + if name == "" { + continue + } + if role == "Member" || (role == "" && len(indices) > 0) { + members = append(members, name) + } + } + return members +} + +// getDB fetches the Postgres CR unstructured, resolving the namespace from the +// factory's kubeconfig flags. +func getDB(ctx context.Context, f cmdutil.Factory, name string) (*unstructured.Unstructured, dynamic.Interface, string, error) { + ns, _, err := f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil, nil, "", err + } + cfg, err := f.ToRESTConfig() + if err != nil { + return nil, nil, "", err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, nil, "", err + } + db, err := dyn.Resource(PostgresGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, nil, "", fmt.Errorf("failed to get Postgres %s/%s: %w", ns, name, err) + } + return db, dyn, ns, nil +} + +// annotateDB merge-patches one annotation onto the Postgres CR (nil value removes). +func annotateDB(ctx context.Context, dyn dynamic.Interface, ns, name, key string, value *string) error { + var v any + if value != nil { + v = *value + } + patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:%s}}}`, key, jsonValue(v)) + _, err := dyn.Resource(PostgresGVR).Namespace(ns).Patch(ctx, name, mergePatchType, []byte(patch), metav1.PatchOptions{}) + return err +} + +func jsonValue(v any) string { + if v == nil { + return "null" + } + return fmt.Sprintf("%q", v) +} + +// requireDistributed prints a loud warning when the database is not a DC-DR +// distributed one; the annotations are harmless there but will do nothing. +func requireDistributed(out func(string, ...any), db *unstructured.Unstructured) { + distributed, _, _ := unstructured.NestedBool(db.Object, "spec", "distributed") + if !distributed { + out("WARNING: %s/%s has spec.distributed=false; DC-DR commands have no effect on it\n", db.GetNamespace(), db.GetName()) + } +} + +// markerConfigMap creates or deletes a human-owned marker ConfigMap (break-glass +// override or standby-hold) in the coordination namespace of the CURRENT cluster, +// which for these markers must be the target DC's spoke. +func markerConfigMap(ctx context.Context, f cmdutil.Factory, ns, name string, remove bool) (string, error) { + cs, err := f.KubernetesClientSet() + if err != nil { + return "", err + } + if remove { + if err := cs.CoreV1().ConfigMaps(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return "", err + } + return "deleted", nil + } + _, err = cs.CoreV1().ConfigMaps(ns).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}}, metav1.CreateOptions{}) + if err != nil { + return "", err + } + return "created", nil +} diff --git a/pkg/dcdr/debug.go b/pkg/dcdr/debug.go new file mode 100644 index 000000000..d31a6dbe9 --- /dev/null +++ b/pkg/dcdr/debug.go @@ -0,0 +1,504 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdDebug is the `dc-dr debug` group: per-symptom diagnosis walkers. +func NewCmdDebug(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "debug", + Short: i18n.T("Diagnose DC-DR symptoms: failover not happening, switchover stuck, fenced database"), + Long: templates.LongDesc(`Walk the DC-DR decision chain and report, in order, every condition that would stop the thing you are waiting for.`), + Run: func(cmd *cobra.Command, args []string) {}, + DisableFlagsInUseLine: true, + DisableAutoGenTag: true, + } + cmd.AddCommand( + newCmdDebugFailover(f), + newCmdDebugSwitchover(f), + newCmdDebugFence(f), + ) + return cmd +} + +type finding struct { + ok bool + title string + detail string + remedy string + blocker bool +} + +func (fd finding) print(w io.Writer) { + mark := "OK " + if !fd.ok { + mark = "FAIL" + if !fd.blocker { + mark = "WARN" + } + } + _, _ = fmt.Fprintf(w, " [%s] %s\n", mark, fd.title) + if fd.detail != "" { + _, _ = fmt.Fprintf(w, " %s\n", fd.detail) + } + if !fd.ok && fd.remedy != "" { + _, _ = fmt.Fprintf(w, " -> %s\n", fd.remedy) + } +} + +// newCmdDebugFailover answers "the active DC is gone / my database is down and +// nothing failed over". +func newCmdDebugFailover(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + cmd := &cobra.Command{ + Use: "failover DB_NAME", + Short: i18n.T("Diagnose why a cross-DC failover is not happening"), + Long: templates.LongDesc(` + Walks every gate between "something is wrong" and "another data center is + primary", reporting which one is holding: + + 1. is the scope registered at all (no PlacementPolicy failoverPolicy + means no Lease, so nothing can ever move); + 2. does the Lease exist, who holds it, and is it still being renewed + (a renewed Lease means the holder's agent is alive: by design NO + database-level condition, client errors, QPS, lag, or a crashed + postgres, ever moves it); + 3. is a break-glass pin or a standby-hold blocking the move; + 4. is the RPO budget holding the promotion (the accept remedy); + 5. are there stale or failed ForceFailOver ops, or a tripped retry cap, + that make the hub skip evaluation. + + KUBECONFIG: the hub cluster; the coordination plane via --coord-*.`), + Example: templates.Examples(` + kubectl dba dc-dr debug failover pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + out := cmd.OutOrStdout() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + _, _ = fmt.Fprintf(out, "Diagnosing failover for %s/%s\n\n", ns, args[0]) + + var findings []finding + + // 1. scope registration + distributed, _, _ := unstructured.NestedBool(db.Object, "spec", "distributed") + dcdrOn := db.GetAnnotations()["dr.kubedb.com/enabled"] == "true" + switch { + case !distributed: + findings = append(findings, finding{title: "database is DC-DR distributed", detail: "spec.distributed is false", remedy: "DC-DR does not apply to this database", blocker: true}) + case !dcdrOn: + findings = append(findings, finding{title: "DC-DR is armed on the database", detail: "annotation dr.kubedb.com/enabled is not \"true\"", remedy: "set dr.kubedb.com/enabled=true; without it the per-DC substrate and the coordinator fence are not configured", blocker: true}) + default: + findings = append(findings, finding{ok: true, title: "database is DC-DR distributed and armed"}) + } + if strings.Contains(scope.Source, "WARNING") { + findings = append(findings, finding{title: "failover scope is registered", detail: scope.Source, remedy: "register the scope in the PlacementPolicy (clusterSpreadConstraint.failoverPolicy.trigger); an unregistered scope has no Lease and no protection", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("failover scope resolves to %s", scope.LeaseName), detail: scope.Source}) + } + + // 2. the Lease itself + coord, cerr := cf.CoordClient(ctx, f) + if cerr != nil { + findings = append(findings, finding{title: "coordination plane reachable", detail: cerr.Error(), remedy: "pass --coord-kubeconfig / --coord-kubeconfig-secret; without the Lease no diagnosis of the authority is possible", blocker: true}) + printFindings(out, findings) + return nil + } + lease, lerr := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if lerr != nil { + findings = append(findings, finding{title: "primary-DC Lease exists", detail: lerr.Error(), remedy: "the topology controller creates it from the PlacementPolicy; check the dr-controlplane topology deployment", blocker: true}) + printFindings(out, findings) + return nil + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + dur := int32(45) + if lease.Spec.LeaseDurationSeconds != nil { + dur = *lease.Spec.LeaseDurationSeconds + } + var age time.Duration + if lease.Spec.RenewTime != nil { + age = time.Since(lease.Spec.RenewTime.Time).Round(time.Second) + } + switch { + case holder == "": + findings = append(findings, finding{ok: true, title: "Lease is unheld", detail: "no data center currently holds it; the first healthy Member to contend will acquire it"}) + case age <= time.Duration(dur)*time.Second: + findings = append(findings, finding{ + title: fmt.Sprintf("holder %q is renewing normally, so the authority will NOT move on its own", holder), + detail: fmt.Sprintf("the Lease was renewed %s ago, inside its %ds duration: that data center's agent is alive and healthy", age, dur), + remedy: "this is by design: no database-level condition (client errors, QPS, lag, a crashed postgres) ever moves the Lease. If the DATABASE is down but the DC is alive, either let the DC's own raft promote a local peer, or move the scope deliberately: kubectl dba dc-dr handoff " + args[0] + " -n " + ns + " --to --yes", + }) + default: + findings = append(findings, finding{ok: true, title: fmt.Sprintf("Lease is EXPIRED (holder %q last renewed %s ago, duration %ds)", holder, age, dur), detail: "a healthy Member DC should acquire it within one retry tick"}) + } + + // 3. pins + if pin := lease.Annotations["dr.open-cluster-management.io/override-hold"]; pin != "" { + findings = append(findings, finding{title: "no break-glass pin is blocking the move", detail: fmt.Sprintf("scope is PINNED to %q", pin), remedy: "remove that DC's override ConfigMap: kubectl dba dc-dr pin-primary --scope " + scope.LeaseName + " --remove --yes (run against that DC's spoke)", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: "no break-glass pin on the Lease"}) + } + if ho := lease.Annotations[AnnLeaseHandoffTo]; ho != "" { + findings = append(findings, finding{title: "no handoff is stuck in flight", detail: fmt.Sprintf("handoff-to=%q is still set, so the target has not acquired yet", ho), remedy: "if the target is standby-held it will never take it: kubectl dba dc-dr pin-standby --scope " + scope.LeaseName + " --remove --yes (against that DC's spoke)"}) + } + findings = append(findings, checkStandbyHolds(ctx, coord, cf.LeaseNS, scope, holder)...) + + // 4. the RPO budget + protected, protSet, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected") + protMsg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage") + if protSet && !protected { + findings = append(findings, finding{ + title: "promotion is not held by the RPO budget", + detail: fmt.Sprintf("protected=false: %s", protMsg), + remedy: "if this is a real failover and the loss is acceptable: kubectl dba dc-dr accept-data-loss " + args[0] + " -n " + ns + " --yes", + }) + } else if protSet { + findings = append(findings, finding{ok: true, title: "protection is confirmed (RPO budget satisfied)"}) + } + if db.GetAnnotations()[AnnAcceptDataLoss] != "" { + findings = append(findings, finding{ok: true, title: "a data-loss acceptance is currently set", detail: "the budget is bypassed for the promotion it authorizes; the operator removes it once that failover lands"}) + } + + // 5. ops objects and conditions + findings = append(findings, checkFailoverOps(ctx, f, ns, args[0])...) + findings = append(findings, checkConditions(db)...) + + printFindings(out, findings) + _, _ = fmt.Fprintf(out, "\nAlso useful:\n") + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr status %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr active-dc %s -n %s\n", args[0], ns) + return nil + }, + } + AddCoordFlags(cmd, &cf) + return cmd +} + +// checkStandbyHolds reports Member DCs whose health Lease is stale, which is the +// only cross-DC-visible hint that a DC cannot take over. The standby-hold marker +// itself is spoke-local and deliberately invisible from here. +func checkStandbyHolds(ctx context.Context, coord kubernetes.Interface, ns string, scope *Scope, holder string) []finding { + members := scope.MemberDCs + if len(members) == 0 { + return nil + } + var out []finding + for _, dc := range members { + if dc == holder { + continue + } + hl, err := coord.CoordinationV1().Leases(ns).Get(ctx, "dc-health-"+dc, metav1.GetOptions{}) + if err != nil { + out = append(out, finding{title: fmt.Sprintf("candidate DC %q is reporting health", dc), detail: err.Error(), remedy: "a DC with no health Lease has no running agent; it cannot acquire the primary role"}) + continue + } + if hl.Spec.RenewTime != nil { + age := time.Since(hl.Spec.RenewTime.Time).Round(time.Second) + dur := int32(15) + if hl.Spec.LeaseDurationSeconds != nil { + dur = *hl.Spec.LeaseDurationSeconds + } + if age > 3*time.Duration(dur)*time.Second { + out = append(out, finding{title: fmt.Sprintf("candidate DC %q is healthy", dc), detail: fmt.Sprintf("its health Lease is stale (%s old)", age), remedy: "that DC's agent is down or cannot reach the coordination plane; it cannot take over until it returns"}) + continue + } + out = append(out, finding{ok: true, title: fmt.Sprintf("candidate DC %q is alive (health renewed %s ago)", dc, age), detail: "if it still refuses to promote, check for a standby-hold ConfigMap on ITS spoke: kubectl -n " + ns + " get cm " + scope.LeaseName + StandbyHoldCMSuffix}) + } + } + return out +} + +func checkFailoverOps(ctx context.Context, f cmdutil.Factory, ns, dbName string) []finding { + cfg, err := f.ToRESTConfig() + if err != nil { + return nil + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil + } + list, err := dyn.Resource(PgOpsGVR).Namespace(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil + } + var stale, failed, progressing []string + for i := range list.Items { + o := &list.Items[i] + dbRef, _, _ := unstructured.NestedString(o.Object, "spec", "databaseRef", "name") + typ, _, _ := unstructured.NestedString(o.Object, "spec", "type") + if dbRef != dbName || typ != "ForceFailOver" { + continue + } + phase, _, _ := unstructured.NestedString(o.Object, "status", "phase") + switch phase { + case "Failed": + failed = append(failed, o.GetName()) + case "Skipped", "Successful": + if time.Since(o.GetCreationTimestamp().Time) > time.Hour { + stale = append(stale, o.GetName()) + } + case "Progressing", "": + progressing = append(progressing, o.GetName()) + } + } + var out []finding + if len(progressing) > 0 { + out = append(out, finding{ok: true, title: "a ForceFailOver is in progress", detail: strings.Join(progressing, ", ")}) + } + if len(failed) > 0 { + out = append(out, finding{title: "no failed ForceFailOver ops are blocking", detail: "failed: " + strings.Join(failed, ", "), remedy: "read their status for the real cause, then delete them; repeated failures trip the retry cap and the hub stops minting new ones"}) + } + if len(stale) > 0 { + out = append(out, finding{title: "no stale ForceFailOver ops are confusing the hub", detail: "old and completed: " + strings.Join(stale, ", "), remedy: "delete them; a stale op targeting the same DC makes the hub read \"already promoted\" and skip evaluation entirely"}) + } + if len(out) == 0 { + out = append(out, finding{ok: true, title: "no stale or failed ForceFailOver ops"}) + } + return out +} + +func checkConditions(db *unstructured.Unstructured) []finding { + conds, _, _ := unstructured.NestedSlice(db.Object, "status", "conditions") + var out []finding + for _, c := range conds { + cm, ok := c.(map[string]any) + if !ok { + continue + } + typ, _ := cm["type"].(string) + status, _ := cm["status"].(string) + msg, _ := cm["message"].(string) + switch typ { + case "ForceFailOverRetryCapReached": + if status == "True" { + out = append(out, finding{title: "the ForceFailOver retry cap is not tripped", detail: msg, remedy: "the hub has stopped minting new failover ops after repeated failures. Fix the underlying failure, delete the failed ops, and it resumes", blocker: true}) + } + case "DCDRPromotionStalled": + if status == "True" { + out = append(out, finding{title: "no stalled promotion is reported", detail: msg, remedy: "the holder cannot promote; check the coordinator logs on that DC's leader pod"}) + } + case "DCDRFailoverScopeShared": + if status == "True" { + out = append(out, finding{ok: true, title: "NOTE: this scope is SHARED", detail: msg + " (every database in it fails over together)"}) + } + } + } + return out +} + +func printFindings(w io.Writer, fs []finding) { + blocking := 0 + for _, fd := range fs { + fd.print(w) + if !fd.ok && fd.blocker { + blocking++ + } + } + _, _ = fmt.Fprintf(w, "\n%d blocking condition(s) found.\n", blocking) +} + +// newCmdDebugSwitchover explains a planned switchover that will not complete. +func newCmdDebugSwitchover(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "switchover DB_NAME", + Short: i18n.T("Diagnose a planned switchover that is not completing"), + Long: templates.LongDesc(` + Reports which of the switchover's gates is holding and why, including the + one that surprises people most: every gate measures by dialing the ACTIVE + primary, so a switchover cannot proceed while that primary is unreachable. + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(`kubectl dba dc-dr debug switchover pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + out := cmd.OutOrStdout() + db, _, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + ann := db.GetAnnotations() + target := ann[AnnSwitchoverTo] + if target == "" && ann[AnnQuiesceActive] == "" { + _, _ = fmt.Fprintf(out, "No switchover is in flight on %s/%s.\n", ns, args[0]) + _, _ = fmt.Fprintf(out, "Start one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) + return nil + } + var findings []finding + if ann[AnnSwitchoverAbort] != "" { + findings = append(findings, finding{title: "no abort is pending", detail: "the abort annotation is set; the hub is unwinding this switchover", remedy: "wait for the annotations to clear, then retry the switchover"}) + } + if started := ann[AnnSwitchoverStart]; started != "" { + if t, perr := time.Parse(time.RFC3339, started); perr == nil { + el := time.Since(t).Round(time.Second) + fd := finding{ok: true, title: fmt.Sprintf("switchover started %s ago", el)} + if el > 10*time.Minute { + fd = finding{title: "switchover is within its timeout", detail: fmt.Sprintf("running for %s, past the default 10m", el), remedy: "it auto-aborts and restores writes to the original active DC; watch for the annotations to clear"} + } + findings = append(findings, fd) + } + } + activeDC, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + dcs, _, _ := unstructured.NestedSlice(db.Object, "status", "disasterRecovery", "dataCenters") + var targetFound bool + for _, d := range dcs { + dm, ok := d.(map[string]any) + if !ok { + continue + } + name, _ := dm["clusterName"].(string) + if name == target { + targetFound = true + healthy, _ := dm["healthy"].(bool) + if !healthy { + findings = append(findings, finding{title: fmt.Sprintf("target %q is healthy", target), detail: "its health Lease is not fresh", remedy: "the switchover refuses an unhealthy target; fix that DC's agent first", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("target %q is healthy", target)}) + } + if lag, ok := toInt64(dm["lagBytes"]); ok { + if lag > zeroRPOLagBytes { + findings = append(findings, finding{title: "target has caught up to the frozen LSN", detail: fmt.Sprintf("lag is %d bytes, needs <= %d", lag, zeroRPOLagBytes), remedy: "this resolves itself once the quiesce freezes the primary and the target replays; if it never shrinks, the target is not streaming"}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("target lag is %d bytes, within the zero-RPO tolerance", lag)}) + } + } else { + findings = append(findings, finding{title: "target lag is known", detail: "no lagBytes reported for the target", remedy: "the hub measures lag by dialing the ACTIVE primary; an unreachable primary makes this permanently unknown and the switchover can never start. Use the failover path instead: dc-dr handoff", blocker: true}) + } + } + if name == activeDC { + if w, ok := dm["writable"].(bool); ok && !w { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("quiesce is IN EFFECT on %q (write-locked)", activeDC)}) + } else if ann[AnnQuiesceActive] == "true" { + findings = append(findings, finding{title: fmt.Sprintf("quiesce has taken effect on %q", activeDC), detail: "the quiesce was requested but the active DC still reads writable", remedy: "the write-lock is confirmed by dialing the active primary; if that primary is down or unreachable this never flips and the switchover stalls. Abort (dc-dr abort) and use dc-dr handoff instead", blocker: true}) + } + } + } + if target != "" && !targetFound { + findings = append(findings, finding{title: fmt.Sprintf("target %q is a known data center", target), detail: "it is not present in status.disasterRecovery.dataCenters", remedy: "check the target name against the PlacementPolicy's Member DCs", blocker: true}) + } + printFindings(out, findings) + _, _ = fmt.Fprintf(out, "\n kubectl dba dc-dr status %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + return nil + }, + } + return cmd +} + +// newCmdDebugFence explains a database that is up but refusing writes. +func newCmdDebugFence(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + cmd := &cobra.Command{ + Use: "fence DB_NAME", + Short: i18n.T("Diagnose a database whose primary is fenced read-only"), + Long: templates.LongDesc(` + A DC-DR database goes read-only by design when its local marker is + missing, stale past its TTL, or names another data center, so that at most + one data center is ever writable. This reports which of those applies, + whether the authority itself is healthy, and what to do when the + coordination plane is the thing that is broken. + + KUBECONFIG: the hub cluster; the coordination plane via --coord-*.`), + Example: templates.Examples(`kubectl dba dc-dr debug fence pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + out := cmd.OutOrStdout() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + var findings []finding + coord, cerr := cf.CoordClient(ctx, f) + if cerr != nil { + findings = append(findings, finding{title: "coordination plane reachable", detail: cerr.Error(), remedy: "if the coordination plane is genuinely down, every DC fences read-only after the marker TTL plus its uncertainty hold. To keep the current primary writable through the outage, pin it: kubectl dba dc-dr pin-primary --scope " + scope.LeaseName + " --yes (run against that DC's spoke)", blocker: true}) + printFindings(out, findings) + return nil + } + lease, lerr := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if lerr != nil { + findings = append(findings, finding{title: "primary-DC Lease exists", detail: lerr.Error(), remedy: "with no Lease there is no marker to project, so every DC fences. Check the dr-controlplane topology controller", blocker: true}) + printFindings(out, findings) + return nil + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + if holder == "" { + findings = append(findings, finding{title: "the scope has an active data center", detail: "the Lease is currently unheld, so every DC's marker says nobody is active and all of them fence", remedy: "a healthy Member acquires within a retry tick; if none does, check that at least one DC's agent is running", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("the authority says %q is active", holder)}) + } + if lease.Spec.RenewTime != nil { + age := time.Since(lease.Spec.RenewTime.Time).Round(time.Second) + if age > 30*time.Second { + findings = append(findings, finding{title: "the authority is being renewed", detail: fmt.Sprintf("last renewed %s ago; the marker projected onto each spoke goes stale after 30s and the fence closes fail-closed", age), remedy: "the holder's agent cannot write to the coordination plane. Fix that, or pin the primary to ride out the outage: kubectl dba dc-dr pin-primary --scope " + scope.LeaseName + " --yes", blocker: true}) + } else { + findings = append(findings, finding{ok: true, title: fmt.Sprintf("the authority is fresh (renewed %s ago)", age)}) + } + } + activeDC, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + if activeDC != "" && holder != "" && activeDC != holder { + findings = append(findings, finding{title: "the database agrees with the authority", detail: fmt.Sprintf("status says %q, the Lease says %q", activeDC, holder), remedy: "the status trails the Lease by a reconcile; if it persists, the hub operator is not reconciling this database"}) + } + _, _ = fmt.Fprintf(out, "Fence diagnosis for %s/%s (scope %s)\n\n", ns, args[0], scope.LeaseName) + printFindings(out, findings) + _, _ = fmt.Fprintf(out, "\nThe marker each data center actually reads lives on its OWN spoke:\n") + _, _ = fmt.Fprintf(out, " kubectl --kubeconfig -n %s get cm %s -o jsonpath='{.data}'\n", cf.LeaseNS, scope.LeaseName) + return nil + }, + } + AddCoordFlags(cmd, &cf) + return cmd +} diff --git a/pkg/dcdr/handoff.go b/pkg/dcdr/handoff.go new file mode 100644 index 000000000..fa9e2a936 --- /dev/null +++ b/pkg/dcdr/handoff.go @@ -0,0 +1,143 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewCmdHandoff moves the failover authority by annotating the Lease itself. Acts +// on the COORDINATION plane (and on the hub only to resolve a database's scope). +func NewCmdHandoff(f cmdutil.Factory) *cobra.Command { + var cf CoordFlags + var leaseName, to string + var yes bool + cmd := &cobra.Command{ + Use: "handoff (DB_NAME | --lease NAME) --to DC", + Short: i18n.T("Move the failover authority for a scope by handing off its primary-DC Lease"), + Long: templates.LongDesc(` + Writes dr.open-cluster-management.io/handoff-to on the scope's primary-DC + Lease. The holding data center's agent releases the Lease once, the target + acquires it within a retry tick, and the annotation clears itself. + + This is the scope-local FAILOVER lever, and the correct tool when the + active data center's database is down but its data center is alive: no + quiesce and no catch-up wait happen, so loss is bounded by the RPO budget + rather than zero. For a healthy primary prefer "dc-dr switchover", which + is zero-RPO. + + It moves EVERY database sharing the scope. Do NOT stop a DC's agent to + force a failover instead: one agent serves every scope its DC holds, so + that expires all of them together. + + KUBECONFIG: the hub cluster (to resolve a database's scope and read the + coordination kubeconfig Secret). The Lease is written on the coordination + plane via the --coord-* flags.`), + Example: templates.Examples(` + # Fail a database's scope over to dc-b + kubectl dba dc-dr handoff pg-dcdr -n demo --to dc-b --yes + + # Move a scope by Lease name (works with no database left) + kubectl dba dc-dr handoff --lease primary-dc-orders --to dc-a --yes`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if (len(args) == 0) == (leaseName == "") { + return fmt.Errorf("give exactly one of: a database name, or --lease") + } + if to == "" { + return fmt.Errorf("--to is required (the target data center)") + } + ctx := context.Background() + out := cmd.OutOrStdout() + scope := &Scope{LeaseName: leaseName, Source: "--lease flag"} + if len(args) == 1 { + db, dyn, _, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err = ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + if len(scope.MemberDCs) > 0 && !slices.Contains(scope.MemberDCs, to) { + return fmt.Errorf("%q is not a Member data center of this database (members: %v)", to, scope.MemberDCs) + } + } + coord, err := cf.CoordClient(ctx, f) + if err != nil { + return err + } + lease, err := coord.CoordinationV1().Leases(cf.LeaseNS).Get(ctx, scope.LeaseName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to read Lease %s/%s: %w", cf.LeaseNS, scope.LeaseName, err) + } + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + if holder == to { + _, _ = fmt.Fprintf(out, "No-op: %s already holds %s.\n", to, scope.LeaseName) + return nil + } + if members := lease.Annotations[AnnLeaseMemberDCs]; members != "" { + if !slices.Contains(strings.Split(members, ","), to) { + return fmt.Errorf("%q is not listed in the Lease's Member DCs (%s); only a Member can hold the primary role", to, members) + } + } + if pin := lease.Annotations["dr.open-cluster-management.io/override-hold"]; pin != "" && pin != to { + return fmt.Errorf("scope %s is PINNED to %q by a break-glass override; remove that DC's override ConfigMap first (kubectl dba dc-dr pin-primary --remove), or the handoff cannot complete", scope.LeaseName, pin) + } + if !yes { + _, _ = fmt.Fprintf(out, "Would move %s from %s to %s. Every database in this scope fails over together, without a quiesce (loss bounded by the RPO budget, not zero).\n", scope.LeaseName, orNone(holder), to) + return fmt.Errorf("re-run with --yes to proceed") + } + // Re-requesting the SAME target must still fire an agent-visible event. + // A merge patch that writes an identical value does not bump the + // resourceVersion, so the informers see nothing and the handoff never + // starts: exactly what happens when a target that was standby-held is + // released and the pending handoff has to be re-driven. Clear first, then + // set, so there is always a real transition to observe. + if lease.Annotations[AnnLeaseHandoffTo] == to { + clear := fmt.Sprintf(`{"metadata":{"annotations":{%q:null}}}`, AnnLeaseHandoffTo) + if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, scope.LeaseName, types.MergePatchType, []byte(clear), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to clear the stale handoff annotation on %s/%s: %w", cf.LeaseNS, scope.LeaseName, err) + } + } + patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:%q}}}`, AnnLeaseHandoffTo, to) + if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, scope.LeaseName, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to annotate Lease %s/%s: %w", cf.LeaseNS, scope.LeaseName, err) + } + _, _ = fmt.Fprintf(out, "Handoff of %s requested: %s -> %s.\n", scope.LeaseName, orNone(holder), to) + _, _ = fmt.Fprintf(out, "The holder releases within seconds and %s acquires on its next retry tick; the annotation clears itself.\n", to) + _, _ = fmt.Fprintf(out, "Verify: kubectl dba dc-dr active-dc --lease %s\n", scope.LeaseName) + _, _ = fmt.Fprintf(out, "NOTE: if the target is held by a standby-hold ConfigMap it will refuse; clear it with dc-dr pin-standby --remove.\n") + return nil + }, + } + cmd.Flags().StringVar(&leaseName, "lease", "", "Act on this Lease directly instead of resolving a database's scope") + cmd.Flags().StringVar(&to, "to", "", "Target data center") + cmd.Flags().BoolVar(&yes, "yes", false, "Confirm the handoff (it moves every database in the scope)") + AddCoordFlags(cmd, &cf) + return cmd +} diff --git a/pkg/dcdr/pin.go b/pkg/dcdr/pin.go new file mode 100644 index 000000000..d9210c21e --- /dev/null +++ b/pkg/dcdr/pin.go @@ -0,0 +1,203 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +type pinKind int + +const ( + pinPrimary pinKind = iota + pinStandby +) + +// NewCmdPin builds either pin-primary (break-glass override: this DC stays primary +// and stays writable even with the control plane gone) or pin-standby +// (standby-hold: this DC never promotes). +// +// Both act on a SPOKE cluster: the marker ConfigMap is human-owned and lives on the +// data center it governs, which is exactly why it still works when the hub is +// unreachable. So the ordinary kubeconfig flags must point at that DC's spoke. +func NewCmdPin(f cmdutil.Factory, kind pinKind) *cobra.Command { + var scopeName, dbName string + var remove, force, yes bool + var cf CoordFlags + + use, short, long, example := pinPrimaryTexts() + suffix := OverrideCMSuffix + if kind == pinStandby { + use, short, long, example = pinStandbyTexts() + suffix = StandbyHoldCMSuffix + } + + cmd := &cobra.Command{ + Use: use, + Short: i18n.T(short), + Long: templates.LongDesc(long), + Example: templates.Examples(example), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if (scopeName == "") == (dbName == "") { + return fmt.Errorf("give exactly one of --scope (a primary-DC Lease name) or --db (resolve the scope from a database)") + } + ctx := context.Background() + out := cmd.OutOrStdout() + lease := scopeName + if dbName != "" { + db, dyn, _, err := getDB(ctx, f, dbName) + if err != nil { + return fmt.Errorf("resolving --db (note: with --db the kubeconfig must reach the hub, while creating the marker needs the SPOKE; pass --scope when running against a spoke): %w", err) + } + s, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + lease = s.LeaseName + _, _ = fmt.Fprintf(out, "Resolved scope %s from database %s (%s).\n", lease, dbName, s.Source) + } + cmName := lease + suffix + if !remove && !yes { + if kind == pinPrimary { + _, _ = fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) + _, _ = fmt.Fprintf(out, "This is a STANDING BYPASS of split-brain protection: the local leader stays writable regardless of what the failover authority says, and the scope cannot fail over while it exists.\n") + } else { + _, _ = fmt.Fprintf(out, "Would create ConfigMap %s/%s on the CURRENT cluster; this data center will never promote while it exists.\n", cf.LeaseNS, cmName) + } + return fmt.Errorf("re-run with --yes to proceed (or --remove --yes to clear an existing pin)") + } + if remove && !yes { + _, _ = fmt.Fprintf(out, "Would delete ConfigMap %s/%s on the CURRENT cluster.\n", cf.LeaseNS, cmName) + return fmt.Errorf("re-run with --yes to proceed") + } + action, err := markerConfigMap(ctx, f, cf.LeaseNS, cmName, remove) + if err != nil { + if !remove || !force { + return err + } + // The pinned data center is dead or unreachable: exactly the situation + // --force exists for. The ConfigMap on its spoke is unreachable garbage; + // what actually blocks the failover is the override-hold annotation on + // the Lease, which only that DC's own (dead) agent would ever clear. + _, _ = fmt.Fprintf(out, "Could not remove ConfigMap %s/%s on the current cluster (%v); proceeding to clear the Lease annotation because --force is set.\n", cf.LeaseNS, cmName, err) + action = "unreachable, skipped" + } + _, _ = fmt.Fprintf(out, "ConfigMap %s/%s %s on the current cluster.\n", cf.LeaseNS, cmName, action) + if kind == pinPrimary && remove && force { + coord, cerr := cf.CoordClient(ctx, f) + if cerr != nil { + return fmt.Errorf("--force needs the coordination plane to clear the override-hold annotation, but it is unreachable: %w", cerr) + } + patch := `{"metadata":{"annotations":{"dr.open-cluster-management.io/override-hold":null}}}` + if _, err := coord.CoordinationV1().Leases(cf.LeaseNS).Patch(ctx, lease, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to clear override-hold on Lease %s/%s: %w", cf.LeaseNS, lease, err) + } + _, _ = fmt.Fprintf(out, "Cleared the override-hold annotation on Lease %s/%s directly (the pinned DC's agent is not alive to do it).\n", cf.LeaseNS, lease) + _, _ = fmt.Fprintf(out, "Members now contend normally; the surviving DC can acquire once its holds are removed.\n") + } + if kind == pinPrimary { + if remove { + _, _ = fmt.Fprintf(out, "The break-glass pin is cleared; normal contention resumes and the override-hold annotation drops from the Lease within seconds.\n") + } else { + _, _ = fmt.Fprintf(out, "This data center is now PINNED primary for scope %s:\n", lease) + _, _ = fmt.Fprintf(out, " - its agent mirrors the pin onto the Lease, so no other Member contends;\n") + _, _ = fmt.Fprintf(out, " - its coordinator keeps the local leader writable even if the marker goes stale (a control-plane outage no longer fences it).\n") + _, _ = fmt.Fprintf(out, " IMPORTANT: only honored on the scope's LAST KNOWN HOLDER. On any other DC the agent refuses it and logs why; it never promotes a standby.\n") + _, _ = fmt.Fprintf(out, " IMPORTANT: while pinned, that DC dying means NO failover happens. Remove the pin as soon as the emergency ends:\n") + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr pin-primary --scope %s --remove --yes\n", lease) + } + } else { + if remove { + _, _ = fmt.Fprintf(out, "The standby-hold is cleared. NOTE: this DC resumes contending on the NEXT Lease event; if the Lease is idle, a pending handoff may take until the agent's informer resync. Touching the Lease (for example dc-dr handoff --to ) makes it immediate.\n") + } else { + _, _ = fmt.Fprintf(out, "This data center is now HELD as a standby for scope %s: it never contends for the Lease, never promotes, and refuses destructive cross-DC rewinds of its data.\n", lease) + _, _ = fmt.Fprintf(out, " It is ignored while this DC is the ACTIVE one (demoting the active DC without a quiesce is unsafe): move the primary away with a switchover first.\n") + _, _ = fmt.Fprintf(out, " Remove with: kubectl dba dc-dr pin-standby --scope %s --remove --yes\n", lease) + } + } + return nil + }, + } + cmd.Flags().StringVar(&scopeName, "scope", "", "Primary-DC Lease name of the scope (for example primary-dc or primary-dc-orders)") + cmd.Flags().StringVar(&dbName, "db", "", "Resolve the scope from this database instead (requires the kubeconfig to reach the hub)") + cmd.Flags().BoolVar(&remove, "remove", false, "Remove the pin instead of creating it") + if kind == pinPrimary { + cmd.Flags().BoolVar(&force, "force", false, "With --remove: also clear the override-hold annotation directly on the Lease, for when the pinned DC is dead and its agent cannot clear it (run against any live cluster; needs the --coord-* flags to reach the coordination plane)") + AddCoordFlags(cmd, &cf) + } else { + cmd.Flags().StringVar(&cf.LeaseNS, "coord-namespace", DefaultCoordNamespace, "Namespace on this spoke that holds the marker ConfigMaps") + } + cmd.Flags().BoolVar(&yes, "yes", false, "Confirm") + return cmd +} + +func pinPrimaryTexts() (use, short, long, example string) { + return "pin-primary (--scope LEASE | --db DB_NAME)", + "Pin this data center as primary (break-glass override): no failover, writable through a control-plane outage", + `Creates the human-owned break-glass override ConfigMap -override in + the coordination namespace of the CURRENT cluster, which must be the data + center you are pinning (its own spoke). + + Two effects, both live-proven: this DC's agent mirrors the pin onto the + Lease so every other Member defers permanently, and this DC's coordinator + forces its leader ACTIVE regardless of marker state, so a sustained + coordination-plane outage no longer fences it read-only after the usual + marker TTL plus uncertainty hold. + + Use it when the failover authority is unreachable and the surviving + primary must keep accepting writes, or as a deliberate "never fail this + scope over" policy. While it stands there is no split-brain protection for + the scope, and nothing takes over if this DC dies. + + KUBECONFIG: the SPOKE of the data center being pinned.`, + `# Keep dc-b primary for the global scope, come what may (run against dc-b) + kubectl dba dc-dr pin-primary --scope primary-dc --yes --kubeconfig ~/.kube/dc-b.yaml + + # Clear it once the emergency is over + kubectl dba dc-dr pin-primary --scope primary-dc --remove --yes --kubeconfig ~/.kube/dc-b.yaml` +} + +func pinStandbyTexts() (use, short, long, example string) { + return "pin-standby (--scope LEASE | --db DB_NAME)", + "Pin this data center as a standby (standby-hold): it never promotes", + `Creates the human-owned standby-hold ConfigMap -standby-hold in the + coordination namespace of the CURRENT cluster, which must be the data + center you are holding down. + + While it exists that DC never contends for the scope's primary-DC Lease, + never promotes (it refuses even an explicit handoff naming it), and its + coordinator refuses destructive cross-DC rewinds or re-seeds of the data + it holds. It fails CLOSED: if the ConfigMap cannot be read the hold is + assumed, so a flaky apiserver never silently drops the protection. + + It is deliberately ignored on the data center that is currently ACTIVE, + because demoting the active DC without a quiesce is unsafe; move the + primary away with a planned switchover first. + + KUBECONFIG: the SPOKE of the data center being held.`, + `# Never let dc-a take the primary role for this scope (run against dc-a) + kubectl dba dc-dr pin-standby --scope primary-dc --yes --kubeconfig ~/.kube/dc-a.yaml + + # Release it + kubectl dba dc-dr pin-standby --scope primary-dc --remove --yes --kubeconfig ~/.kube/dc-a.yaml` +} diff --git a/pkg/dcdr/status.go b/pkg/dcdr/status.go new file mode 100644 index 000000000..7820ea496 --- /dev/null +++ b/pkg/dcdr/status.go @@ -0,0 +1,303 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +// stepState is the rendered state of one switchover step. +type stepState int + +const ( + stepDone stepState = iota + stepCurrent + stepPending + stepSkipped +) + +func (s stepState) mark() string { + switch s { + case stepDone: + return "[done] " + case stepCurrent: + return "[NOW] " + case stepSkipped: + return "[n/a] " + default: + return "[pending]" + } +} + +// NewCmdStatus renders a one-shot picture of a database's DC-DR state and, when a +// switchover is in flight, which of its steps are complete and what happens next. +// Deliberately NOT a watch: it prints once and exits, so the user re-runs it. +func NewCmdStatus(f cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "status DB_NAME", + Short: i18n.T("Show DC-DR state and switchover progress once (re-run to see further progress)"), + Long: templates.LongDesc(` + Prints the database's failover scope, per data center state, protection + verdict, and, when a planned switchover is in flight, its step-by-step + progress: what has completed, what is happening NOW, and what remains. + + One-shot by design: it does not follow. Run it again to see the next + state, which keeps its output readable in tickets and transcripts. + + KUBECONFIG: the hub cluster.`), + Example: templates.Examples(` + kubectl dba dc-dr status pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + out := cmd.OutOrStdout() + ann := db.GetAnnotations() + dbPhase, _, _ := unstructured.NestedString(db.Object, "status", "phase") + activeDC, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC") + drPhase, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "phase") + protected, protectedSet, _ := unstructured.NestedBool(db.Object, "status", "disasterRecovery", "protected") + protMsg, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "protectionMessage") + + _, _ = fmt.Fprintf(out, "Database: %s/%s (%s)\n", ns, args[0], dbPhase) + _, _ = fmt.Fprintf(out, "Failover scope: %s\n (%s)\n", scope.LeaseName, scope.Source) + if len(scope.MemberDCs) > 0 { + _, _ = fmt.Fprintf(out, "Member DCs: %s\n", strings.Join(scope.MemberDCs, ", ")) + } + _, _ = fmt.Fprintf(out, "Active DC: %s DR phase: %s\n", orNone(activeDC), orNone(drPhase)) + if protectedSet { + _, _ = fmt.Fprintf(out, "Protected: %v", protected) + if protMsg != "" { + _, _ = fmt.Fprintf(out, " (%s)", protMsg) + } + _, _ = fmt.Fprintln(out) + } + + // Per-DC table. + dcs, found, _ := unstructured.NestedSlice(db.Object, "status", "disasterRecovery", "dataCenters") + if found && len(dcs) > 0 { + _, _ = fmt.Fprintf(out, "\nData centers:\n") + _, _ = fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", "NAME", "ROLE", "WRITABLE", "HEALTHY", "LAG(BYTES)", "STREAMER") + for _, d := range dcs { + dm, ok := d.(map[string]any) + if !ok { + continue + } + name, _ := dm["clusterName"].(string) + role, _ := dm["role"].(string) + writable := boolCell(dm, "writable") + healthy := boolCell(dm, "healthy") + lag := "-" + if v, ok := dm["lagBytes"]; ok { + lag = fmt.Sprintf("%v", v) + } + streamer, _ := dm["crossDCStreamer"].(string) + _, _ = fmt.Fprintf(out, " %-10s %-9s %-9s %-9s %-12s %s\n", name, orNone(role), writable, healthy, lag, orNone(streamer)) + } + } + + // Switchover progress, only when one is in flight or was just requested. + target := ann[AnnSwitchoverTo] + quiesced := ann[AnnQuiesceActive] == "true" + aborting := ann[AnnSwitchoverAbort] != "" + started := ann[AnnSwitchoverStart] + if target != "" || quiesced || aborting { + _, _ = fmt.Fprintf(out, "\nPlanned switchover") + if target != "" { + _, _ = fmt.Fprintf(out, " to %q", target) + } + if started != "" { + if t, perr := time.Parse(time.RFC3339, started); perr == nil { + _, _ = fmt.Fprintf(out, ", started %s ago", time.Since(t).Round(time.Second)) + } + } + _, _ = fmt.Fprintln(out, ":") + if aborting { + _, _ = fmt.Fprintf(out, " ABORT requested (%s is set). The hub clears the quiesce and restores writes to %s.\n", AnnSwitchoverAbort, orNone(activeDC)) + _, _ = fmt.Fprintf(out, " Re-run this command until the switchover annotations are gone and DR phase is Steady.\n") + return nil + } + renderSwitchoverSteps(out, target, activeDC, quiesced, dcs) + _, _ = fmt.Fprintf(out, "\n Abort: kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + _, _ = fmt.Fprintf(out, " Re-run this command to see the next step; it does not follow.\n") + return nil + } + + _, _ = fmt.Fprintf(out, "\nNo switchover in flight.\n") + if drPhase == "FailingOver" { + _, _ = fmt.Fprintf(out, "DR phase is FailingOver with no switchover annotation, so this is an UNPLANNED failover.\n") + _, _ = fmt.Fprintf(out, " If it is not completing: kubectl dba dc-dr debug failover %s -n %s\n", args[0], ns) + } + if protectedSet && !protected { + _, _ = fmt.Fprintf(out, "Protection is NOT confirmed. If a promotion is held by the RPO budget:\n") + _, _ = fmt.Fprintf(out, " kubectl dba dc-dr accept-data-loss %s -n %s --yes\n", args[0], ns) + } + _, _ = fmt.Fprintf(out, "Trigger one: kubectl dba dc-dr switchover %s -n %s --to \n", args[0], ns) + return nil + }, + } + return cmd +} + +// renderSwitchoverSteps prints the operator's switchover sequence with each step's +// state derived from the SAME status fields the operator's own gates read, so the +// output cannot drift from the real decision: +// +// 1 target validated (healthy, lag known and within the switchover budget) +// 2 quiesce requested (annotation set) +// 3 quiesce in effect (active DC's writable flipped false) +// 4 target caught up (target lagBytes <= 8 KiB) +// 5 Lease handed off (activeDC == target) +// 6 old DC demoted, annotations cleared, phase Steady +func renderSwitchoverSteps(out interface{ Write([]byte) (int, error) }, target, activeDC string, quiesced bool, dcs []any) { + var targetLag any + targetHealthy, targetKnown := false, false + activeWritable, activeWritableKnown := true, false + for _, d := range dcs { + dm, ok := d.(map[string]any) + if !ok { + continue + } + name, _ := dm["clusterName"].(string) + if name == target { + targetKnown = true + targetHealthy, _ = dm["healthy"].(bool) + targetLag = dm["lagBytes"] + } + if name == activeDC { + if w, ok := dm["writable"].(bool); ok { + activeWritable, activeWritableKnown = w, true + } + } + } + handedOff := target != "" && activeDC == target + + step1 := stepPending + switch { + case !targetKnown: + step1 = stepCurrent + case targetHealthy && targetLag != nil: + step1 = stepDone + default: + step1 = stepCurrent + } + step2 := stepPending + if quiesced { + step2 = stepDone + } + step3 := stepPending + switch { + case handedOff: + step3 = stepDone + case quiesced && activeWritableKnown && !activeWritable: + step3 = stepDone + case quiesced: + step3 = stepCurrent + } + step4 := stepPending + caughtUp := false + if lag, ok := toInt64(targetLag); ok && lag <= zeroRPOLagBytes { + caughtUp = true + } + switch { + case handedOff: + step4 = stepDone + case step3 == stepDone && caughtUp: + step4 = stepDone + case step3 == stepDone: + step4 = stepCurrent + } + step5 := stepPending + if handedOff { + step5 = stepDone + } else if step4 == stepDone { + step5 = stepCurrent + } + step6 := stepPending + if handedOff { + step6 = stepCurrent + } + + _, _ = fmt.Fprintf(out, " %s 1. target %q validated: healthy and lag known, within the switchover budget\n", step1.mark(), target) + _, _ = fmt.Fprintf(out, " %s 2. quiesce requested on the active DC (%s)\n", step2.mark(), orNone(activeDC)) + _, _ = fmt.Fprintf(out, " %s 3. quiesce IN EFFECT: active primary write-locked, its LSN frozen\n", step3.mark()) + lagText := "unknown" + if targetLag != nil { + lagText = fmt.Sprintf("%v bytes", targetLag) + } + _, _ = fmt.Fprintf(out, " %s 4. target caught up to the frozen LSN (now %s, needs <= %d)\n", step4.mark(), lagText, zeroRPOLagBytes) + _, _ = fmt.Fprintf(out, " %s 5. primary-DC Lease handed off to %q\n", step5.mark(), target) + _, _ = fmt.Fprintf(out, " %s 6. old DC demoted to standby, annotations cleared, DR phase back to Steady\n", step6.mark()) + + _, _ = fmt.Fprintf(out, "\n NEXT: ") + switch { + case step1 == stepCurrent: + _, _ = fmt.Fprintf(out, "waiting for the target's health and lag to be observable. If it never becomes healthy the switchover cannot start.\n") + case step3 == stepCurrent: + _, _ = fmt.Fprintf(out, "waiting for the write-lock to take hold on %s. This needs the active primary to be UP and reachable; a dead primary can never satisfy it (use the failover path).\n", orNone(activeDC)) + case step4 == stepCurrent: + _, _ = fmt.Fprintf(out, "waiting for %q to replay the last %s. This is the only step whose duration depends on your write volume.\n", target, lagText) + case step5 == stepCurrent: + _, _ = fmt.Fprintf(out, "handing off the Lease; the target promotes within seconds.\n") + case step6 == stepCurrent: + _, _ = fmt.Fprintf(out, "the Lease has moved to %q. The old DC self-fences and re-cascades, then the annotations clear and the phase returns to Steady.\n", target) + default: + _, _ = fmt.Fprintf(out, "the operator picks it up on its next reconcile.\n") + } +} + +func toInt64(v any) (int64, bool) { + switch n := v.(type) { + case int64: + return n, true + case float64: + return int64(n), true + case int: + return int64(n), true + } + return 0, false +} + +func boolCell(m map[string]any, key string) string { + if v, ok := m[key].(bool); ok { + return fmt.Sprintf("%v", v) + } + return "-" +} + +func orNone(s string) string { + if s == "" { + return "" + } + return s +} diff --git a/pkg/dcdr/switchover.go b/pkg/dcdr/switchover.go new file mode 100644 index 000000000..4baf8c7a3 --- /dev/null +++ b/pkg/dcdr/switchover.go @@ -0,0 +1,100 @@ +/* +Copyright AppsCode Inc. and Contributors +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dcdr + +import ( + "context" + "fmt" + "slices" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +const mergePatchType = types.MergePatchType + +// NewCmdSwitchover triggers a planned, zero-RPO switchover by annotating the +// database. Acts on the HUB cluster (ordinary kubeconfig flags). +func NewCmdSwitchover(f cmdutil.Factory) *cobra.Command { + var to string + cmd := &cobra.Command{ + Use: "switchover DB_NAME --to DC", + Short: i18n.T("Trigger a planned zero-RPO switchover of a distributed database to another data center"), + Long: templates.LongDesc(` + Sets the dr.kubedb.com/switchover-to annotation on the Postgres. The hub + operator then quiesces the active primary (write-locked), waits for the + target data center to catch up to the frozen LSN, hands off the primary-DC + Lease, and clears the annotation. Requires the active primary to be up and + accepting connections: the safety gates measure by dialing it and fail + closed, so a dead primary cannot be switched away from (use the failover + path instead: dc-dr handoff, and dc-dr accept-data-loss if the RPO budget + holds it). + + KUBECONFIG: the hub cluster (where the Postgres CR lives).`), + Example: templates.Examples(` + # Move demo/pg-dcdr to data center dc-a, with zero data loss + kubectl dba dc-dr switchover pg-dcdr -n demo --to dc-a + + # Watch the progress (one-shot, run repeatedly) + kubectl dba dc-dr status pg-dcdr -n demo`), + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("exactly one database name is required") + } + if to == "" { + return fmt.Errorf("--to is required (the target data center)") + } + ctx := context.Background() + db, dyn, ns, err := getDB(ctx, f, args[0]) + if err != nil { + return err + } + requireDistributed(printfErr(cmd), db) + + scope, err := ResolveScopeForDB(ctx, dyn, db) + if err != nil { + return err + } + if len(scope.MemberDCs) > 0 && !slices.Contains(scope.MemberDCs, to) { + return fmt.Errorf("%q is not a Member data center of this database (members: %v); an Arbiter or Witness DC can never become primary", to, scope.MemberDCs) + } + if active, _, _ := unstructured.NestedString(db.Object, "status", "disasterRecovery", "activeDC"); active == to { + cmd.Printf("No-op: %s is already the active data center of %s/%s.\n", to, ns, args[0]) + return nil + } + if err := annotateDB(ctx, dyn, ns, args[0], AnnSwitchoverTo, &to); err != nil { + return err + } + cmd.Printf("Switchover of %s/%s to %q requested (scope %s, from %s).\n", ns, args[0], to, scope.LeaseName, scope.Source) + cmd.Printf("The operator will quiesce, wait for catch-up, and hand off; zero committed rows are lost.\n") + cmd.Printf("Monitor: kubectl dba dc-dr status %s -n %s\n", args[0], ns) + cmd.Printf("Abort: kubectl dba dc-dr abort %s -n %s\n", args[0], ns) + if scope.LeaseName == GlobalPrimaryLease { + cmd.Printf("NOTE: this database follows the GLOBAL scope; every database in that scope switches with it.\n") + } + return nil + }, + } + cmd.Flags().StringVar(&to, "to", "", "Target data center (must be a Member DC of the database's PlacementPolicy)") + return cmd +} + +func printfErr(cmd *cobra.Command) func(string, ...any) { + return func(format string, a ...any) { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), format, a...) } +}